2024-11-14 02:43:20,764 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 02:43:20,779 main DEBUG Took 0.012519 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 02:43:20,779 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 02:43:20,779 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 02:43:20,781 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 02:43:20,782 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:20,794 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 02:43:20,811 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:20,814 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:20,815 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:20,816 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:20,817 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:20,817 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:20,819 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:20,819 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:20,820 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:20,821 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:20,822 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:20,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:20,824 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:20,824 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:20,825 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:20,826 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:20,826 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:20,827 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:20,828 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:20,828 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:20,829 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:20,830 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:20,831 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:20,831 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:43:20,832 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:20,832 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 02:43:20,835 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:43:20,837 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 02:43:20,840 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 02:43:20,841 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 02:43:20,843 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 02:43:20,844 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 02:43:20,857 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 02:43:20,861 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 02:43:20,864 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 02:43:20,864 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 02:43:20,865 main DEBUG createAppenders(={Console}) 2024-11-14 02:43:20,866 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-14 02:43:20,867 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 02:43:20,867 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-14 02:43:20,868 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 02:43:20,869 main DEBUG OutputStream closed 2024-11-14 02:43:20,869 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 02:43:20,870 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 02:43:20,870 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-14 02:43:20,982 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 02:43:20,985 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 02:43:20,986 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 02:43:20,988 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 02:43:20,989 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 02:43:20,990 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 02:43:20,990 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 02:43:20,991 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 02:43:20,991 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 02:43:20,992 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 02:43:20,992 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 02:43:20,993 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 02:43:20,993 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 02:43:20,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 02:43:20,997 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 02:43:20,998 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 02:43:20,998 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 02:43:21,000 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 02:43:21,004 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 02:43:21,005 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-14 02:43:21,005 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 02:43:21,006 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-14T02:43:21,341 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802 2024-11-14 02:43:21,344 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 02:43:21,344 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T02:43:21,352 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-14T02:43:21,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T02:43:21,394 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/cluster_d5add8ed-0ad9-3315-92f0-5e7efe2147ab, deleteOnExit=true 2024-11-14T02:43:21,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T02:43:21,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/test.cache.data in system properties and HBase conf 2024-11-14T02:43:21,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T02:43:21,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/hadoop.log.dir in system properties and HBase conf 2024-11-14T02:43:21,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T02:43:21,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T02:43:21,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T02:43:21,534 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T02:43:21,685 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:43:21,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T02:43:21,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T02:43:21,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T02:43:21,695 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T02:43:21,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T02:43:21,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T02:43:21,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T02:43:21,698 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T02:43:21,699 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T02:43:21,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/nfs.dump.dir in system properties and HBase conf 2024-11-14T02:43:21,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/java.io.tmpdir in system properties and HBase conf 2024-11-14T02:43:21,701 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T02:43:21,702 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T02:43:21,703 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T02:43:22,904 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T02:43:22,997 INFO [Time-limited test {}] log.Log(170): Logging initialized @3069ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T02:43:23,088 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T02:43:23,168 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T02:43:23,192 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T02:43:23,192 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T02:43:23,194 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T02:43:23,211 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T02:43:23,217 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/hadoop.log.dir/,AVAILABLE} 2024-11-14T02:43:23,219 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T02:43:23,408 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/java.io.tmpdir/jetty-localhost-34673-hadoop-hdfs-3_4_1-tests_jar-_-any-17937240400278920538/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T02:43:23,413 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:34673} 2024-11-14T02:43:23,414 INFO [Time-limited test {}] server.Server(415): Started @3487ms 2024-11-14T02:43:23,966 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T02:43:23,974 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T02:43:23,975 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T02:43:23,975 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T02:43:23,976 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T02:43:23,976 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/hadoop.log.dir/,AVAILABLE} 2024-11-14T02:43:23,977 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T02:43:24,075 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/java.io.tmpdir/jetty-localhost-33487-hadoop-hdfs-3_4_1-tests_jar-_-any-3218980634846592402/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T02:43:24,076 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:33487} 2024-11-14T02:43:24,076 INFO [Time-limited test {}] server.Server(415): Started @4149ms 2024-11-14T02:43:24,123 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T02:43:25,076 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/cluster_d5add8ed-0ad9-3315-92f0-5e7efe2147ab/data/data2/current/BP-1718068783-172.17.0.2-1731552202398/current, will proceed with Du for space computation calculation, 2024-11-14T02:43:25,076 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/cluster_d5add8ed-0ad9-3315-92f0-5e7efe2147ab/data/data1/current/BP-1718068783-172.17.0.2-1731552202398/current, will proceed with Du for space computation calculation, 2024-11-14T02:43:25,107 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T02:43:25,147 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4881a672a9325052 with lease ID 0xc196c93cbfea60e2: Processing first storage report for DS-738c4717-b73a-4f1f-ab95-c8962b4c6568 from datanode DatanodeRegistration(127.0.0.1:41091, datanodeUuid=a1e01ff2-a378-43eb-b94e-1163f3e76400, infoPort=34469, infoSecurePort=0, ipcPort=40235, storageInfo=lv=-57;cid=testClusterID;nsid=251763354;c=1731552202399) 2024-11-14T02:43:25,148 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4881a672a9325052 with lease ID 0xc196c93cbfea60e2: from storage DS-738c4717-b73a-4f1f-ab95-c8962b4c6568 node DatanodeRegistration(127.0.0.1:41091, datanodeUuid=a1e01ff2-a378-43eb-b94e-1163f3e76400, infoPort=34469, infoSecurePort=0, ipcPort=40235, storageInfo=lv=-57;cid=testClusterID;nsid=251763354;c=1731552202399), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T02:43:25,148 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4881a672a9325052 with lease ID 0xc196c93cbfea60e2: Processing first storage report for DS-e9d16969-08ba-4ccc-814e-18b3d6eded85 from datanode DatanodeRegistration(127.0.0.1:41091, datanodeUuid=a1e01ff2-a378-43eb-b94e-1163f3e76400, infoPort=34469, infoSecurePort=0, ipcPort=40235, storageInfo=lv=-57;cid=testClusterID;nsid=251763354;c=1731552202399) 2024-11-14T02:43:25,148 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4881a672a9325052 with lease ID 0xc196c93cbfea60e2: from storage DS-e9d16969-08ba-4ccc-814e-18b3d6eded85 node DatanodeRegistration(127.0.0.1:41091, datanodeUuid=a1e01ff2-a378-43eb-b94e-1163f3e76400, infoPort=34469, infoSecurePort=0, ipcPort=40235, storageInfo=lv=-57;cid=testClusterID;nsid=251763354;c=1731552202399), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T02:43:25,197 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802 2024-11-14T02:43:25,260 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/cluster_d5add8ed-0ad9-3315-92f0-5e7efe2147ab/zookeeper_0, clientPort=62229, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/cluster_d5add8ed-0ad9-3315-92f0-5e7efe2147ab/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/cluster_d5add8ed-0ad9-3315-92f0-5e7efe2147ab/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T02:43:25,272 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62229 2024-11-14T02:43:25,283 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T02:43:25,286 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T02:43:25,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741825_1001 (size=7) 2024-11-14T02:43:25,943 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1 with version=8 2024-11-14T02:43:25,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/hbase-staging 2024-11-14T02:43:26,040 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T02:43:26,300 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e5dc8b5cf5e9:0 server-side Connection retries=45 2024-11-14T02:43:26,309 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:43:26,310 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T02:43:26,316 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T02:43:26,316 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:43:26,316 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T02:43:26,443 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T02:43:26,492 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T02:43:26,500 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T02:43:26,503 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T02:43:26,524 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7199 (auto-detected) 2024-11-14T02:43:26,525 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T02:43:26,540 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38717 2024-11-14T02:43:26,560 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38717 connecting to ZooKeeper ensemble=127.0.0.1:62229 2024-11-14T02:43:26,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:387170x0, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T02:43:26,631 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38717-0x1013739d5650000 connected 2024-11-14T02:43:26,713 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T02:43:26,716 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T02:43:26,726 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:43:26,731 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1, hbase.cluster.distributed=false 2024-11-14T02:43:26,759 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T02:43:26,764 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38717 2024-11-14T02:43:26,765 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38717 2024-11-14T02:43:26,765 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38717 2024-11-14T02:43:26,766 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38717 2024-11-14T02:43:26,767 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38717 2024-11-14T02:43:26,858 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e5dc8b5cf5e9:0 server-side Connection retries=45 2024-11-14T02:43:26,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:43:26,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T02:43:26,860 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T02:43:26,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:43:26,861 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T02:43:26,863 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T02:43:26,865 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T02:43:26,866 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46387 2024-11-14T02:43:26,868 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46387 connecting to ZooKeeper ensemble=127.0.0.1:62229 2024-11-14T02:43:26,869 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T02:43:26,871 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T02:43:26,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:463870x0, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T02:43:26,880 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:463870x0, quorum=127.0.0.1:62229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:43:26,880 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46387-0x1013739d5650001 connected 2024-11-14T02:43:26,884 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T02:43:26,891 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T02:43:26,894 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T02:43:26,900 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T02:43:26,901 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46387 2024-11-14T02:43:26,902 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46387 2024-11-14T02:43:26,902 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46387 2024-11-14T02:43:26,903 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46387 2024-11-14T02:43:26,903 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46387 2024-11-14T02:43:26,921 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e5dc8b5cf5e9:38717 2024-11-14T02:43:26,922 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:43:26,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:43:26,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:43:26,934 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:43:26,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T02:43:26,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:26,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:26,965 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T02:43:26,966 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e5dc8b5cf5e9,38717,1731552206127 from backup master directory 2024-11-14T02:43:26,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:43:26,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:43:26,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:43:26,975 WARN [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T02:43:26,975 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:43:26,978 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T02:43:26,979 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T02:43:27,030 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/hbase.id] with ID: f6372a0b-df7a-4882-85af-70267aaacec2 2024-11-14T02:43:27,031 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/.tmp/hbase.id 2024-11-14T02:43:27,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741826_1002 (size=42) 2024-11-14T02:43:27,446 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/.tmp/hbase.id]:[hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/hbase.id] 2024-11-14T02:43:27,495 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T02:43:27,501 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T02:43:27,520 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-14T02:43:27,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:27,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:27,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741827_1003 (size=196) 2024-11-14T02:43:28,003 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:43:28,005 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T02:43:28,019 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:28,022 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T02:43:28,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741828_1004 (size=1189) 2024-11-14T02:43:28,470 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store 2024-11-14T02:43:28,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741829_1005 (size=34) 2024-11-14T02:43:28,498 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T02:43:28,501 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:43:28,502 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T02:43:28,503 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:43:28,503 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:43:28,505 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T02:43:28,505 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:43:28,505 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:43:28,507 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731552208502Disabling compacts and flushes for region at 1731552208502Disabling writes for close at 1731552208505 (+3 ms)Writing region close event to WAL at 1731552208505Closed at 1731552208505 2024-11-14T02:43:28,509 WARN [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/.initializing 2024-11-14T02:43:28,510 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/WALs/e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:43:28,520 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T02:43:28,536 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e5dc8b5cf5e9%2C38717%2C1731552206127, suffix=, logDir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/WALs/e5dc8b5cf5e9,38717,1731552206127, archiveDir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/oldWALs, maxLogs=10 2024-11-14T02:43:28,566 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/WALs/e5dc8b5cf5e9,38717,1731552206127/e5dc8b5cf5e9%2C38717%2C1731552206127.1731552208543, exclude list is [], retry=0 2024-11-14T02:43:28,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41091,DS-738c4717-b73a-4f1f-ab95-c8962b4c6568,DISK] 2024-11-14T02:43:28,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T02:43:28,632 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/WALs/e5dc8b5cf5e9,38717,1731552206127/e5dc8b5cf5e9%2C38717%2C1731552206127.1731552208543 2024-11-14T02:43:28,633 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34469:34469)] 2024-11-14T02:43:28,634 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:43:28,634 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:43:28,638 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:28,639 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:28,679 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:28,703 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T02:43:28,707 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:28,710 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:28,711 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:28,714 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T02:43:28,715 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:28,716 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:28,716 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:28,720 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T02:43:28,720 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:28,721 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:28,722 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:28,726 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T02:43:28,727 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:28,728 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:28,729 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:28,733 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:28,734 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:28,739 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:28,740 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:28,745 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T02:43:28,750 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:43:28,756 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:43:28,758 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66771896, jitterRate=-0.005021214485168457}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T02:43:28,765 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731552208652Initializing all the Stores at 1731552208655 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552208656 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552208657 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552208657Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552208657Cleaning up temporary data from old regions at 1731552208740 (+83 ms)Region opened successfully at 1731552208765 (+25 ms) 2024-11-14T02:43:28,767 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T02:43:28,803 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d03a7d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e5dc8b5cf5e9/172.17.0.2:0 2024-11-14T02:43:28,839 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T02:43:28,852 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T02:43:28,852 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T02:43:28,856 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T02:43:28,858 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T02:43:28,863 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-14T02:43:28,864 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T02:43:28,886 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T02:43:28,893 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T02:43:28,985 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T02:43:28,990 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T02:43:28,993 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T02:43:29,006 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T02:43:29,008 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T02:43:29,013 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T02:43:29,025 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T02:43:29,028 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T02:43:29,037 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T02:43:29,059 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T02:43:29,068 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T02:43:29,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T02:43:29,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T02:43:29,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:29,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:29,083 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e5dc8b5cf5e9,38717,1731552206127, sessionid=0x1013739d5650000, setting cluster-up flag (Was=false) 2024-11-14T02:43:29,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:29,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:29,142 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T02:43:29,145 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:43:29,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:29,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:29,195 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T02:43:29,199 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:43:29,208 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T02:43:29,213 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(746): ClusterId : f6372a0b-df7a-4882-85af-70267aaacec2 2024-11-14T02:43:29,215 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T02:43:29,229 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T02:43:29,229 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T02:43:29,239 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T02:43:29,239 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fc6b335, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e5dc8b5cf5e9/172.17.0.2:0 2024-11-14T02:43:29,253 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e5dc8b5cf5e9:46387 2024-11-14T02:43:29,256 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T02:43:29,256 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T02:43:29,256 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T02:43:29,259 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(2659): reportForDuty to master=e5dc8b5cf5e9,38717,1731552206127 with port=46387, startcode=1731552206828 2024-11-14T02:43:29,269 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T02:43:29,276 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T02:43:29,286 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T02:43:29,294 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T02:43:29,301 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54315, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T02:43:29,298 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e5dc8b5cf5e9,38717,1731552206127 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T02:43:29,307 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e5dc8b5cf5e9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:43:29,307 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e5dc8b5cf5e9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:43:29,308 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e5dc8b5cf5e9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:43:29,308 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e5dc8b5cf5e9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:43:29,308 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e5dc8b5cf5e9:0, corePoolSize=10, maxPoolSize=10 2024-11-14T02:43:29,308 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:29,308 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e5dc8b5cf5e9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T02:43:29,308 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:29,308 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:29,311 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731552239311 2024-11-14T02:43:29,312 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T02:43:29,313 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T02:43:29,314 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T02:43:29,315 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T02:43:29,317 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T02:43:29,317 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T02:43:29,318 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T02:43:29,318 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T02:43:29,318 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:29,320 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:29,321 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T02:43:29,322 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T02:43:29,323 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T02:43:29,323 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T02:43:29,326 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T02:43:29,326 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T02:43:29,328 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.large.0-1731552209327,5,FailOnTimeoutGroup] 2024-11-14T02:43:29,328 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.small.0-1731552209328,5,FailOnTimeoutGroup] 2024-11-14T02:43:29,329 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:29,329 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T02:43:29,330 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:29,330 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:29,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741831_1007 (size=1321) 2024-11-14T02:43:29,337 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T02:43:29,337 WARN [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T02:43:29,439 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(2659): reportForDuty to master=e5dc8b5cf5e9,38717,1731552206127 with port=46387, startcode=1731552206828 2024-11-14T02:43:29,445 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:29,449 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] master.ServerManager(517): Registering regionserver=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:29,459 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1 2024-11-14T02:43:29,459 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34671 2024-11-14T02:43:29,459 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T02:43:29,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T02:43:29,469 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] zookeeper.ZKUtil(111): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:29,470 WARN [RS:0;e5dc8b5cf5e9:46387 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T02:43:29,470 INFO [RS:0;e5dc8b5cf5e9:46387 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T02:43:29,470 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/WALs/e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:29,472 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e5dc8b5cf5e9,46387,1731552206828] 2024-11-14T02:43:29,497 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T02:43:29,512 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T02:43:29,516 INFO [RS:0;e5dc8b5cf5e9:46387 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T02:43:29,516 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:29,517 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T02:43:29,522 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T02:43:29,523 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:29,523 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:29,523 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:29,523 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:29,523 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:29,524 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:29,524 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e5dc8b5cf5e9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T02:43:29,524 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:29,524 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:29,524 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:29,525 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:29,525 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:29,525 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:43:29,525 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e5dc8b5cf5e9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T02:43:29,525 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T02:43:29,526 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:29,526 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:29,526 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:29,526 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:29,526 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:29,527 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,46387,1731552206828-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T02:43:29,544 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T02:43:29,545 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,46387,1731552206828-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:29,545 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:29,546 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.Replication(171): e5dc8b5cf5e9,46387,1731552206828 started 2024-11-14T02:43:29,562 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:29,562 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1482): Serving as e5dc8b5cf5e9,46387,1731552206828, RpcServer on e5dc8b5cf5e9/172.17.0.2:46387, sessionid=0x1013739d5650001 2024-11-14T02:43:29,563 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T02:43:29,563 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:29,563 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5dc8b5cf5e9,46387,1731552206828' 2024-11-14T02:43:29,563 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T02:43:29,564 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T02:43:29,565 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T02:43:29,565 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T02:43:29,565 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:29,565 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5dc8b5cf5e9,46387,1731552206828' 2024-11-14T02:43:29,565 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T02:43:29,566 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T02:43:29,566 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T02:43:29,566 INFO [RS:0;e5dc8b5cf5e9:46387 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T02:43:29,567 INFO [RS:0;e5dc8b5cf5e9:46387 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T02:43:29,673 INFO [RS:0;e5dc8b5cf5e9:46387 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T02:43:29,677 INFO [RS:0;e5dc8b5cf5e9:46387 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e5dc8b5cf5e9%2C46387%2C1731552206828, suffix=, logDir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/WALs/e5dc8b5cf5e9,46387,1731552206828, archiveDir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/oldWALs, maxLogs=32 2024-11-14T02:43:29,692 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/WALs/e5dc8b5cf5e9,46387,1731552206828/e5dc8b5cf5e9%2C46387%2C1731552206828.1731552209679, exclude list is [], retry=0 2024-11-14T02:43:29,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41091,DS-738c4717-b73a-4f1f-ab95-c8962b4c6568,DISK] 2024-11-14T02:43:29,700 INFO [RS:0;e5dc8b5cf5e9:46387 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/WALs/e5dc8b5cf5e9,46387,1731552206828/e5dc8b5cf5e9%2C46387%2C1731552206828.1731552209679 2024-11-14T02:43:29,700 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34469:34469)] 2024-11-14T02:43:29,736 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T02:43:29,736 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1 2024-11-14T02:43:29,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741833_1009 (size=32) 2024-11-14T02:43:30,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:43:30,160 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T02:43:30,163 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T02:43:30,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:30,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:30,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T02:43:30,166 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T02:43:30,167 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:30,167 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:30,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T02:43:30,170 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T02:43:30,170 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:30,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:30,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T02:43:30,174 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T02:43:30,174 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:30,175 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:30,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T02:43:30,176 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740 2024-11-14T02:43:30,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740 2024-11-14T02:43:30,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T02:43:30,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T02:43:30,180 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:43:30,182 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T02:43:30,186 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:43:30,187 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61792971, jitterRate=-0.0792129784822464}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:43:30,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731552210155Initializing all the Stores at 1731552210160 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552210160Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552210160Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552210160Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552210160Cleaning up temporary data from old regions at 1731552210179 (+19 ms)Region opened successfully at 1731552210190 (+11 ms) 2024-11-14T02:43:30,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T02:43:30,190 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T02:43:30,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T02:43:30,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T02:43:30,191 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T02:43:30,192 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T02:43:30,192 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731552210190Disabling compacts and flushes for region at 1731552210190Disabling writes for close at 1731552210191 (+1 ms)Writing region close event to WAL at 1731552210191Closed at 1731552210192 (+1 ms) 2024-11-14T02:43:30,195 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T02:43:30,195 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T02:43:30,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T02:43:30,208 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T02:43:30,210 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T02:43:30,368 DEBUG [e5dc8b5cf5e9:38717 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T02:43:30,379 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:30,386 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e5dc8b5cf5e9,46387,1731552206828, state=OPENING 2024-11-14T02:43:30,479 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T02:43:30,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:30,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:43:30,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:43:30,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:43:30,493 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T02:43:30,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:43:30,674 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T02:43:30,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44289, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T02:43:30,686 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T02:43:30,687 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T02:43:30,687 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T02:43:30,691 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e5dc8b5cf5e9%2C46387%2C1731552206828.meta, suffix=.meta, logDir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/WALs/e5dc8b5cf5e9,46387,1731552206828, archiveDir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/oldWALs, maxLogs=32 2024-11-14T02:43:30,705 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/WALs/e5dc8b5cf5e9,46387,1731552206828/e5dc8b5cf5e9%2C46387%2C1731552206828.meta.1731552210693.meta, exclude list is [], retry=0 2024-11-14T02:43:30,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41091,DS-738c4717-b73a-4f1f-ab95-c8962b4c6568,DISK] 2024-11-14T02:43:30,712 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/WALs/e5dc8b5cf5e9,46387,1731552206828/e5dc8b5cf5e9%2C46387%2C1731552206828.meta.1731552210693.meta 2024-11-14T02:43:30,713 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34469:34469)] 2024-11-14T02:43:30,713 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:43:30,715 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T02:43:30,717 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T02:43:30,721 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T02:43:30,724 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T02:43:30,725 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:43:30,725 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T02:43:30,725 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T02:43:30,728 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T02:43:30,730 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T02:43:30,730 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:30,730 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:30,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T02:43:30,732 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T02:43:30,732 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:30,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:30,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T02:43:30,734 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T02:43:30,734 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:30,735 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:30,735 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T02:43:30,737 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T02:43:30,737 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:30,738 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:43:30,738 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T02:43:30,739 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740 2024-11-14T02:43:30,742 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740 2024-11-14T02:43:30,744 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T02:43:30,744 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T02:43:30,745 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:43:30,747 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T02:43:30,749 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66893935, jitterRate=-0.003202691674232483}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:43:30,749 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T02:43:30,750 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731552210726Writing region info on filesystem at 1731552210726Initializing all the Stores at 1731552210728 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552210728Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552210728Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552210728Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552210728Cleaning up temporary data from old regions at 1731552210744 (+16 ms)Running coprocessor post-open hooks at 1731552210749 (+5 ms)Region opened successfully at 1731552210750 (+1 ms) 2024-11-14T02:43:30,756 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731552210666 2024-11-14T02:43:30,767 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T02:43:30,768 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T02:43:30,769 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:30,771 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e5dc8b5cf5e9,46387,1731552206828, state=OPEN 2024-11-14T02:43:30,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T02:43:30,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T02:43:30,867 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:43:30,867 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:43:30,867 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:30,877 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T02:43:30,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e5dc8b5cf5e9,46387,1731552206828 in 370 msec 2024-11-14T02:43:30,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T02:43:30,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 680 msec 2024-11-14T02:43:30,888 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T02:43:30,888 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T02:43:30,912 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:30,914 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:43:30,937 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:30,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40285, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:30,961 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7220 sec 2024-11-14T02:43:30,962 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731552210961, completionTime=-1 2024-11-14T02:43:30,964 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T02:43:30,965 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T02:43:30,997 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-14T02:43:30,998 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731552270998 2024-11-14T02:43:30,998 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731552330998 2024-11-14T02:43:30,998 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-11-14T02:43:31,001 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,38717,1731552206127-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:31,001 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,38717,1731552206127-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:31,001 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,38717,1731552206127-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:31,003 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e5dc8b5cf5e9:38717, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:31,003 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:31,004 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:31,011 DEBUG [master/e5dc8b5cf5e9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T02:43:31,039 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.064sec 2024-11-14T02:43:31,041 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T02:43:31,042 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T02:43:31,043 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T02:43:31,044 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T02:43:31,044 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T02:43:31,045 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,38717,1731552206127-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T02:43:31,046 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,38717,1731552206127-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T02:43:31,057 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T02:43:31,058 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T02:43:31,058 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,38717,1731552206127-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:43:31,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:31,130 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T02:43:31,130 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T02:43:31,134 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:43:31,137 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:31,145 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:43:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:43:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27add714, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:43:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:31,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:31,156 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:31,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:31,161 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:43:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:31,173 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:31,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:43:31,208 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=215, ProcessCount=11, AvailableMemoryMB=11046 2024-11-14T02:43:31,235 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:43:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:43:31,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@fb2a85e 2024-11-14T02:43:31,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:43:31,254 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:43:31,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:43:31,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:43:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T02:43:31,280 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T02:43:31,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-14T02:43:31,283 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:31,287 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T02:43:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:43:31,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741835_1011 (size=1061) 2024-11-14T02:43:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:43:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:43:31,718 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1 2024-11-14T02:43:31,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741836_1012 (size=53) 2024-11-14T02:43:31,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:43:32,128 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:43:32,129 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b0f2fe4f29c62d92b3ee0b2d165c19cb, disabling compactions & flushes 2024-11-14T02:43:32,129 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:32,129 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:32,129 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. after waiting 0 ms 2024-11-14T02:43:32,130 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:32,130 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:32,130 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: Waiting for close lock at 1731552212129Disabling compacts and flushes for region at 1731552212129Disabling writes for close at 1731552212129Writing region close event to WAL at 1731552212130 (+1 ms)Closed at 1731552212130 2024-11-14T02:43:32,134 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T02:43:32,141 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731552212134"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731552212134"}]},"ts":"1731552212134"} 2024-11-14T02:43:32,145 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T02:43:32,147 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T02:43:32,150 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552212147"}]},"ts":"1731552212147"} 2024-11-14T02:43:32,154 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T02:43:32,156 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b0f2fe4f29c62d92b3ee0b2d165c19cb, ASSIGN}] 2024-11-14T02:43:32,158 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b0f2fe4f29c62d92b3ee0b2d165c19cb, ASSIGN 2024-11-14T02:43:32,159 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b0f2fe4f29c62d92b3ee0b2d165c19cb, ASSIGN; state=OFFLINE, location=e5dc8b5cf5e9,46387,1731552206828; forceNewPlan=false, retain=false 2024-11-14T02:43:32,312 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b0f2fe4f29c62d92b3ee0b2d165c19cb, regionState=OPENING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:32,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b0f2fe4f29c62d92b3ee0b2d165c19cb, ASSIGN because future has completed 2024-11-14T02:43:32,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:43:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:43:32,488 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:32,488 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:43:32,489 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:32,489 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:43:32,490 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:32,490 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:32,492 INFO [StoreOpener-b0f2fe4f29c62d92b3ee0b2d165c19cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:32,496 INFO [StoreOpener-b0f2fe4f29c62d92b3ee0b2d165c19cb-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:43:32,497 INFO [StoreOpener-b0f2fe4f29c62d92b3ee0b2d165c19cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b0f2fe4f29c62d92b3ee0b2d165c19cb columnFamilyName A 2024-11-14T02:43:32,497 DEBUG [StoreOpener-b0f2fe4f29c62d92b3ee0b2d165c19cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:32,498 INFO [StoreOpener-b0f2fe4f29c62d92b3ee0b2d165c19cb-1 {}] regionserver.HStore(327): Store=b0f2fe4f29c62d92b3ee0b2d165c19cb/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:32,498 INFO [StoreOpener-b0f2fe4f29c62d92b3ee0b2d165c19cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:32,500 INFO [StoreOpener-b0f2fe4f29c62d92b3ee0b2d165c19cb-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:43:32,500 INFO [StoreOpener-b0f2fe4f29c62d92b3ee0b2d165c19cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b0f2fe4f29c62d92b3ee0b2d165c19cb columnFamilyName B 2024-11-14T02:43:32,501 DEBUG [StoreOpener-b0f2fe4f29c62d92b3ee0b2d165c19cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:32,501 INFO [StoreOpener-b0f2fe4f29c62d92b3ee0b2d165c19cb-1 {}] regionserver.HStore(327): Store=b0f2fe4f29c62d92b3ee0b2d165c19cb/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:32,502 INFO [StoreOpener-b0f2fe4f29c62d92b3ee0b2d165c19cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:32,504 INFO [StoreOpener-b0f2fe4f29c62d92b3ee0b2d165c19cb-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:43:32,504 INFO [StoreOpener-b0f2fe4f29c62d92b3ee0b2d165c19cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b0f2fe4f29c62d92b3ee0b2d165c19cb columnFamilyName C 2024-11-14T02:43:32,504 DEBUG [StoreOpener-b0f2fe4f29c62d92b3ee0b2d165c19cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:43:32,505 INFO [StoreOpener-b0f2fe4f29c62d92b3ee0b2d165c19cb-1 {}] regionserver.HStore(327): Store=b0f2fe4f29c62d92b3ee0b2d165c19cb/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:43:32,505 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:32,506 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:32,507 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:32,508 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:32,510 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:32,510 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:32,511 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:43:32,514 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:32,517 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:43:32,518 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b0f2fe4f29c62d92b3ee0b2d165c19cb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71430374, jitterRate=0.06439551711082458}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:43:32,518 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:32,520 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: Running coprocessor pre-open hook at 1731552212490Writing region info on filesystem at 1731552212490Initializing all the Stores at 1731552212492 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552212492Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552212492Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552212492Cleaning up temporary data from old regions at 1731552212510 (+18 ms)Running coprocessor post-open hooks at 1731552212518 (+8 ms)Region opened successfully at 1731552212519 (+1 ms) 2024-11-14T02:43:32,522 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., pid=6, masterSystemTime=1731552212478 2024-11-14T02:43:32,525 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:32,525 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:32,526 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b0f2fe4f29c62d92b3ee0b2d165c19cb, regionState=OPEN, openSeqNum=2, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:32,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:43:32,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T02:43:32,538 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 in 209 msec 2024-11-14T02:43:32,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T02:43:32,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b0f2fe4f29c62d92b3ee0b2d165c19cb, ASSIGN in 381 msec 2024-11-14T02:43:32,542 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T02:43:32,542 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552212542"}]},"ts":"1731552212542"} 2024-11-14T02:43:32,545 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T02:43:32,547 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T02:43:32,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2790 sec 2024-11-14T02:43:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:43:33,449 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:33,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:43:33,459 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:33,461 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:43:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:43:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3d2fd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:43:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:33,466 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7951063c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,470 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:33,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:43:33,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:33,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:33,477 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:43:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:43:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@531cc4ce 2024-11-14T02:43:33,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:43:33,482 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:43:33,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,487 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:43:33,487 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:33,489 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:43:33,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:33,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:43:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6d3ad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:43:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:33,492 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:33,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27cf2c02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,494 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:43:33,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:33,495 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:43:33,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:33,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:43:33,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77470c1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:43:33,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:33,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:33,499 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0d5e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:43:33,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:33,503 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:43:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:43:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@230faddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:43:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:33,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:33,507 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e61ecee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:43:33,511 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:33,512 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:43:33,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:33,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:43:33,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14fde2a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:43:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:33,517 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6988c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,519 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:43:33,519 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:33,520 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:43:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:43:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@784361b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:43:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:33,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:33,523 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:33,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d4015bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,526 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:43:33,526 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:33,527 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:43:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:43:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d8477a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:43:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:33,530 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31fc369a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,531 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:43:33,532 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:33,533 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:43:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:43:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a0271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:43:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:33,535 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eca617b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a046d27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,537 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:43:33,537 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:33,539 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:43:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:43:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0d5e57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:43:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:33,542 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:33,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a56f43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ef02a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:43:33,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:43:33,551 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:43:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:43:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:43:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc5965e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:43:33,553 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:43:33,553 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:33,554 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:43:33,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72723b16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:43:33,561 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:33,561 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:33,563 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:33,563 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:43:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:43:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:33,566 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:43:33,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:43:33,567 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:33,567 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:33,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:33,569 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:33,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:33,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:33,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:43:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-14T02:43:33,574 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:43:33,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:43:33,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:43:33,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T02:43:33,578 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:33,581 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:33,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:43:33,583 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:33,585 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:33,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:43:33,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:33,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:33,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:43:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:43:33,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:43:33,597 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:33,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:33,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:43:33,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:33,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:33,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:33,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:43:33,609 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:43:33,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:43:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:43:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:43:33,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:43:33,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:43:33,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:43:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T02:43:33,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:33,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:33,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:33,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:33,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:33,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:33,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:33,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:33,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:33,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:33,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:33,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:33,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:33,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731552213776, max timestamp=1731552213815Num uniques 859; 2024-11-14T02:43:33,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T02:43:33,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731552213776, max timestamp=1731552213820Num uniques 859; 2024-11-14T02:43:33,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T02:43:33,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731552213766, max timestamp=1731552213815Num uniques 859; 2024-11-14T02:43:33,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T02:43:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552273784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:33,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/449dde5541904740a3407373eb48e0dc is 50, key is test_row_0/A:col10/1731552213585/Put/seqid=0 2024-11-14T02:43:33,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741837_1013 (size=12001) 2024-11-14T02:43:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T02:43:33,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552273942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:33,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:33,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:33,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552273947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:33,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552273947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552273967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731552273967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:33,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:33,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:33,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:33,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:33,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:33,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:33,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552274076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:34,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552274077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:34,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552274077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731552274086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:34,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:34,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:34,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:34,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:34,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552274186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T02:43:34,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:34,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552274286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:34,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:34,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/449dde5541904740a3407373eb48e0dc 2024-11-14T02:43:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552274296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:34,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:34,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552274297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:34,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731552274297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:34,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:34,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:34,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:34,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/534a6ac05d5f464aac5382d35fb169c9 is 50, key is test_row_0/B:col10/1731552213585/Put/seqid=0 2024-11-14T02:43:34,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741838_1014 (size=12001) 2024-11-14T02:43:34,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/534a6ac05d5f464aac5382d35fb169c9 2024-11-14T02:43:34,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/d479c901466343e4855dc9353fa9377e is 50, key is test_row_0/C:col10/1731552213585/Put/seqid=0 2024-11-14T02:43:34,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:34,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:34,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:34,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:34,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:34,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552274496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:34,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741839_1015 (size=12001) 2024-11-14T02:43:34,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552274606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:34,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552274606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552274607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731552274617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:34,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:34,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:34,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:34,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:34,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:34,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T02:43:34,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:34,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:34,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:34,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:34,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/d479c901466343e4855dc9353fa9377e 2024-11-14T02:43:34,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/449dde5541904740a3407373eb48e0dc as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/449dde5541904740a3407373eb48e0dc 2024-11-14T02:43:34,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:34,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:34,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/449dde5541904740a3407373eb48e0dc, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T02:43:34,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/534a6ac05d5f464aac5382d35fb169c9 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/534a6ac05d5f464aac5382d35fb169c9 2024-11-14T02:43:34,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/534a6ac05d5f464aac5382d35fb169c9, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T02:43:34,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/d479c901466343e4855dc9353fa9377e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/d479c901466343e4855dc9353fa9377e 2024-11-14T02:43:35,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/d479c901466343e4855dc9353fa9377e, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T02:43:35,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=126.29 KB/129316 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 1341ms, sequenceid=12, compaction requested=false 2024-11-14T02:43:35,011 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T02:43:35,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:35,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=132.99 KB heapSize=382.24 KB 2024-11-14T02:43:35,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:35,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:35,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:35,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:35,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:35,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/fd710b97265c4c44b8c621e3141b56a6 is 50, key is test_row_0/A:col10/1731552215023/Put/seqid=0 2024-11-14T02:43:35,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741840_1016 (size=14341) 2024-11-14T02:43:35,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/fd710b97265c4c44b8c621e3141b56a6 2024-11-14T02:43:35,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:35,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:35,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:35,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:35,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:35,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:35,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/39f11951602245d8acea2a4e9de1dbe0 is 50, key is test_row_0/B:col10/1731552215023/Put/seqid=0 2024-11-14T02:43:35,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741841_1017 (size=12001) 2024-11-14T02:43:35,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/39f11951602245d8acea2a4e9de1dbe0 2024-11-14T02:43:35,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731552275126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/f974d45410ec41d9b68452a26761f179 is 50, key is test_row_0/C:col10/1731552215023/Put/seqid=0 2024-11-14T02:43:35,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741842_1018 (size=12001) 2024-11-14T02:43:35,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552275138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:35,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552275150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552275141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552275158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:35,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:35,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:35,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:35,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:35,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:35,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552275306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552275306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552275308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552275308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:35,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:35,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:35,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:35,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:35,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:35,491 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T02:43:35,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552275546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:35,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552275547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552275549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552275547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:35,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:35,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:35,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:35,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:35,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:35,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:35,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/f974d45410ec41d9b68452a26761f179 2024-11-14T02:43:35,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/fd710b97265c4c44b8c621e3141b56a6 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/fd710b97265c4c44b8c621e3141b56a6 2024-11-14T02:43:35,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/fd710b97265c4c44b8c621e3141b56a6, entries=200, sequenceid=49, filesize=14.0 K 2024-11-14T02:43:35,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/39f11951602245d8acea2a4e9de1dbe0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/39f11951602245d8acea2a4e9de1dbe0 2024-11-14T02:43:35,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/39f11951602245d8acea2a4e9de1dbe0, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T02:43:35,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/f974d45410ec41d9b68452a26761f179 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/f974d45410ec41d9b68452a26761f179 2024-11-14T02:43:35,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/f974d45410ec41d9b68452a26761f179, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T02:43:35,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~132.99 KB/136186, heapSize ~382.20 KB/391368, currentSize=46.96 KB/48090 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 650ms, sequenceid=49, compaction requested=false 2024-11-14T02:43:35,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:35,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T02:43:35,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:35,719 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:43:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T02:43:35,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/13e764296a814371b6c8239a7b04e222 is 50, key is test_row_0/A:col10/1731552215125/Put/seqid=0 2024-11-14T02:43:35,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741843_1019 (size=12001) 2024-11-14T02:43:35,798 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/13e764296a814371b6c8239a7b04e222 2024-11-14T02:43:35,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/d4256978f69746b4bb83ed3c746d97f3 is 50, key is test_row_0/B:col10/1731552215125/Put/seqid=0 2024-11-14T02:43:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:35,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:35,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741844_1020 (size=12001) 2024-11-14T02:43:35,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552215910, max timestamp=1731552215935Num uniques 859; 2024-11-14T02:43:35,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:43:35,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552215910, max timestamp=1731552215937Num uniques 859; 2024-11-14T02:43:35,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:43:35,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552215910, max timestamp=1731552215935Num uniques 859; 2024-11-14T02:43:35,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:43:36,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552275999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552275999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552276003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552276007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552276116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552276117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552276126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552276135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731552276175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,192 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in b0f2fe4f29c62d92b3ee0b2d165c19cb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:43:36,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,236 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T02:43:36,288 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/d4256978f69746b4bb83ed3c746d97f3 2024-11-14T02:43:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/4854ed0a285041a2b04eb28e6972c48f is 50, key is test_row_0/C:col10/1731552215125/Put/seqid=0 2024-11-14T02:43:36,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552276335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552276336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552276336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552276356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741845_1021 (size=12001) 2024-11-14T02:43:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,490 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T02:43:36,490 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T02:43:36,495 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T02:43:36,495 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T02:43:36,496 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T02:43:36,496 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T02:43:36,498 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T02:43:36,498 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T02:43:36,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552276655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552276666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552276666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552276676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:36,768 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/4854ed0a285041a2b04eb28e6972c48f 2024-11-14T02:43:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/13e764296a814371b6c8239a7b04e222 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/13e764296a814371b6c8239a7b04e222 2024-11-14T02:43:36,802 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/13e764296a814371b6c8239a7b04e222, entries=150, sequenceid=59, filesize=11.7 K 2024-11-14T02:43:36,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/d4256978f69746b4bb83ed3c746d97f3 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d4256978f69746b4bb83ed3c746d97f3 2024-11-14T02:43:36,824 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d4256978f69746b4bb83ed3c746d97f3, entries=150, sequenceid=59, filesize=11.7 K 2024-11-14T02:43:36,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/4854ed0a285041a2b04eb28e6972c48f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/4854ed0a285041a2b04eb28e6972c48f 2024-11-14T02:43:36,851 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/4854ed0a285041a2b04eb28e6972c48f, entries=150, sequenceid=59, filesize=11.7 K 2024-11-14T02:43:36,853 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 1134ms, sequenceid=59, compaction requested=true 2024-11-14T02:43:36,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:36,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:36,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-14T02:43:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-14T02:43:36,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T02:43:36,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2700 sec 2024-11-14T02:43:36,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.2970 sec 2024-11-14T02:43:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:37,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=140.89 KB heapSize=400.45 KB 2024-11-14T02:43:37,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731552215980, max timestamp=1731552217183Num uniques 1068; 2024-11-14T02:43:37,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-14T02:43:37,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731552215980, max timestamp=1731552217183Num uniques 1068; 2024-11-14T02:43:37,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-14T02:43:37,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731552215980, max timestamp=1731552217190Num uniques 1068; 2024-11-14T02:43:37,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-11-14T02:43:37,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:37,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:37,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:37,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:37,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:37,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:37,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/125486a6a79244f6b809edd8386f4466 is 50, key is test_row_0/A:col10/1731552217183/Put/seqid=0 2024-11-14T02:43:37,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741846_1022 (size=12001) 2024-11-14T02:43:37,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552217314, max timestamp=1731552217393Num uniques 859; 2024-11-14T02:43:37,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:43:37,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552217314, max timestamp=1731552217393Num uniques 859; 2024-11-14T02:43:37,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:43:37,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552217314, max timestamp=1731552217393Num uniques 859; 2024-11-14T02:43:37,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:43:37,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731552217450, max timestamp=1731552217485Num uniques 1018; 2024-11-14T02:43:37,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731552217450, max timestamp=1731552217485Num uniques 1018; 2024-11-14T02:43:37,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=16848, suffix heap size=133840,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=368 2024-11-14T02:43:37,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=16848, suffix heap size=133840,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=368 2024-11-14T02:43:37,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731552217450, max timestamp=1731552217485Num uniques 1018; 2024-11-14T02:43:37,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T02:43:37,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731552217532, max timestamp=1731552217571Num uniques 1077; 2024-11-14T02:43:37,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731552217532, max timestamp=1731552217571Num uniques 1227; 2024-11-14T02:43:37,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731552217532, max timestamp=1731552217571Num uniques 1227; 2024-11-14T02:43:37,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=56192, new segment data size=14972, suffix heap size=155576,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1227, new segment cells count=327 2024-11-14T02:43:37,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-14T02:43:37,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=56192, new segment data size=14972, suffix heap size=155576,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1227, new segment cells count=327 2024-11-14T02:43:37,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:37,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:37,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552277647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552277650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:37,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552277652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:37,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552277658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/125486a6a79244f6b809edd8386f4466 2024-11-14T02:43:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T02:43:37,736 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:37,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-14T02:43:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T02:43:37,742 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:37,744 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:37,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:37,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/977c3b9088be48a4a4f1b83e6cf15282 is 50, key is test_row_0/B:col10/1731552217183/Put/seqid=0 2024-11-14T02:43:37,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:37,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552277786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:37,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552277787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552277786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552277787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:37,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741847_1023 (size=12001) 2024-11-14T02:43:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T02:43:37,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T02:43:37,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:37,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:37,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:37,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:37,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:38,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552278015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:38,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552278016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552278016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:38,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552278016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T02:43:38,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:38,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:38,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:38,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:38,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T02:43:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:38,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/977c3b9088be48a4a4f1b83e6cf15282 2024-11-14T02:43:38,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T02:43:38,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:38,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:38,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:38,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:38,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:38,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731552278206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:38,228 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in b0f2fe4f29c62d92b3ee0b2d165c19cb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:43:38,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:38,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/334acba7ae8447be9eff06493eb535ae is 50, key is test_row_0/C:col10/1731552217183/Put/seqid=0 2024-11-14T02:43:38,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741848_1024 (size=12001) 2024-11-14T02:43:38,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.38 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/334acba7ae8447be9eff06493eb535ae 2024-11-14T02:43:38,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/125486a6a79244f6b809edd8386f4466 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/125486a6a79244f6b809edd8386f4466 2024-11-14T02:43:38,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552278325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/125486a6a79244f6b809edd8386f4466, entries=150, sequenceid=98, filesize=11.7 K 2024-11-14T02:43:38,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/977c3b9088be48a4a4f1b83e6cf15282 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/977c3b9088be48a4a4f1b83e6cf15282 2024-11-14T02:43:38,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552278337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552278337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552278337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T02:43:38,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/977c3b9088be48a4a4f1b83e6cf15282, entries=150, sequenceid=98, filesize=11.7 K 2024-11-14T02:43:38,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/334acba7ae8447be9eff06493eb535ae as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/334acba7ae8447be9eff06493eb535ae 2024-11-14T02:43:38,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T02:43:38,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:38,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:38,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:38,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:38,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:38,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/334acba7ae8447be9eff06493eb535ae, entries=150, sequenceid=98, filesize=11.7 K 2024-11-14T02:43:38,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.75 KB/33532, heapSize ~83.44 KB/85440, currentSize=154.31 KB/158010 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 1234ms, sequenceid=98, compaction requested=true 2024-11-14T02:43:38,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:38,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:38,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:38,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:38,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:38,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:38,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:38,447 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:38,451 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:38,452 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:38,453 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/B is initiating minor compaction (all files) 2024-11-14T02:43:38,454 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/B in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:38,454 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/534a6ac05d5f464aac5382d35fb169c9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/39f11951602245d8acea2a4e9de1dbe0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d4256978f69746b4bb83ed3c746d97f3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/977c3b9088be48a4a4f1b83e6cf15282] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=46.9 K 2024-11-14T02:43:38,457 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 534a6ac05d5f464aac5382d35fb169c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731552213585 2024-11-14T02:43:38,459 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 39f11951602245d8acea2a4e9de1dbe0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731552213940 2024-11-14T02:43:38,459 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:38,459 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/A is initiating minor compaction (all files) 2024-11-14T02:43:38,459 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/A in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:38,460 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting d4256978f69746b4bb83ed3c746d97f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552215049 2024-11-14T02:43:38,460 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/449dde5541904740a3407373eb48e0dc, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/fd710b97265c4c44b8c621e3141b56a6, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/13e764296a814371b6c8239a7b04e222, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/125486a6a79244f6b809edd8386f4466] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=49.2 K 2024-11-14T02:43:38,461 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 977c3b9088be48a4a4f1b83e6cf15282, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731552215980 2024-11-14T02:43:38,461 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 449dde5541904740a3407373eb48e0dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731552213585 2024-11-14T02:43:38,463 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd710b97265c4c44b8c621e3141b56a6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731552213933 2024-11-14T02:43:38,465 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13e764296a814371b6c8239a7b04e222, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552215049 2024-11-14T02:43:38,466 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 125486a6a79244f6b809edd8386f4466, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731552215980 2024-11-14T02:43:38,518 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#B#compaction#12 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:38,519 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/9c5c5b87768f4803afa791b63bb835b5 is 50, key is test_row_0/B:col10/1731552217183/Put/seqid=0 2024-11-14T02:43:38,521 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#A#compaction#13 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:38,522 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/2c0a765d035047828064ab46a8987035 is 50, key is test_row_0/A:col10/1731552217183/Put/seqid=0 2024-11-14T02:43:38,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T02:43:38,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:38,546 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=154.31 KB heapSize=429 KB 2024-11-14T02:43:38,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:38,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:38,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:38,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:38,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:38,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:38,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741849_1025 (size=12139) 2024-11-14T02:43:38,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741850_1026 (size=12139) 2024-11-14T02:43:38,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/2cd289e7e9834a15a94fd402d3ee0703 is 50, key is test_row_0/A:col10/1731552217651/Put/seqid=0 2024-11-14T02:43:38,607 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/9c5c5b87768f4803afa791b63bb835b5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/9c5c5b87768f4803afa791b63bb835b5 2024-11-14T02:43:38,621 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/2c0a765d035047828064ab46a8987035 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/2c0a765d035047828064ab46a8987035 2024-11-14T02:43:38,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741851_1027 (size=12151) 2024-11-14T02:43:38,642 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/B of b0f2fe4f29c62d92b3ee0b2d165c19cb into 9c5c5b87768f4803afa791b63bb835b5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:38,642 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/A of b0f2fe4f29c62d92b3ee0b2d165c19cb into 2c0a765d035047828064ab46a8987035(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:38,642 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:38,642 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:38,645 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/B, priority=12, startTime=1731552218444; duration=0sec 2024-11-14T02:43:38,645 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/A, priority=12, startTime=1731552218429; duration=0sec 2024-11-14T02:43:38,645 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:38,645 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:38,645 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:B 2024-11-14T02:43:38,645 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:A 2024-11-14T02:43:38,646 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:38,651 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:38,651 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/C is initiating minor compaction (all files) 2024-11-14T02:43:38,651 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/C in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:38,652 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/d479c901466343e4855dc9353fa9377e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/f974d45410ec41d9b68452a26761f179, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/4854ed0a285041a2b04eb28e6972c48f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/334acba7ae8447be9eff06493eb535ae] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=46.9 K 2024-11-14T02:43:38,653 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting d479c901466343e4855dc9353fa9377e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731552213585 2024-11-14T02:43:38,653 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting f974d45410ec41d9b68452a26761f179, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731552213940 2024-11-14T02:43:38,654 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4854ed0a285041a2b04eb28e6972c48f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552215049 2024-11-14T02:43:38,656 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 334acba7ae8447be9eff06493eb535ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731552215980 2024-11-14T02:43:38,697 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#C#compaction#15 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:38,698 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/7c79168ab0a44be1bbca12544587daf8 is 50, key is test_row_0/C:col10/1731552217183/Put/seqid=0 2024-11-14T02:43:38,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741852_1028 (size=12139) 2024-11-14T02:43:38,758 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/7c79168ab0a44be1bbca12544587daf8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/7c79168ab0a44be1bbca12544587daf8 2024-11-14T02:43:38,788 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/C of b0f2fe4f29c62d92b3ee0b2d165c19cb into 7c79168ab0a44be1bbca12544587daf8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:38,788 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:38,788 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/C, priority=12, startTime=1731552218444; duration=0sec 2024-11-14T02:43:38,788 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:38,788 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:C 2024-11-14T02:43:38,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:38,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T02:43:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552278866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:38,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:38,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552278876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552278875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:38,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:38,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552278876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:38,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:39,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552278996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:39,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552278996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:39,043 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/2cd289e7e9834a15a94fd402d3ee0703 2024-11-14T02:43:39,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/7028ef20e4374831a2a6bc98569212da is 50, key is test_row_0/B:col10/1731552217651/Put/seqid=0 2024-11-14T02:43:39,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741853_1029 (size=12151) 2024-11-14T02:43:39,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552279207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:39,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:39,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:39,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:39,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552279215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:39,523 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/7028ef20e4374831a2a6bc98569212da 2024-11-14T02:43:39,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552279525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:39,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:39,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:39,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:39,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552279525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:39,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/3afcd246c260451a90f7b08e98309f71 is 50, key is test_row_0/C:col10/1731552217651/Put/seqid=0 2024-11-14T02:43:39,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741854_1030 (size=12151) 2024-11-14T02:43:39,604 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/3afcd246c260451a90f7b08e98309f71 2024-11-14T02:43:39,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/2cd289e7e9834a15a94fd402d3ee0703 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/2cd289e7e9834a15a94fd402d3ee0703 2024-11-14T02:43:39,649 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/2cd289e7e9834a15a94fd402d3ee0703, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T02:43:39,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/7028ef20e4374831a2a6bc98569212da as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7028ef20e4374831a2a6bc98569212da 2024-11-14T02:43:39,669 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7028ef20e4374831a2a6bc98569212da, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T02:43:39,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/3afcd246c260451a90f7b08e98309f71 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3afcd246c260451a90f7b08e98309f71 2024-11-14T02:43:39,703 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3afcd246c260451a90f7b08e98309f71, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T02:43:39,706 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~428.95 KB/439248, currentSize=33.54 KB/34350 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 1159ms, sequenceid=169, compaction requested=false 2024-11-14T02:43:39,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:39,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:39,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-14T02:43:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-14T02:43:39,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-14T02:43:39,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9650 sec 2024-11-14T02:43:39,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.9810 sec 2024-11-14T02:43:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T02:43:39,896 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:39,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-14T02:43:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T02:43:39,899 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:39,900 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:39,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:39,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:43:39,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:39,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:39,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:39,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:39,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:39,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:39,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/4fa7ed0a3da64386aaf019be8fa167c5 is 50, key is test_row_0/A:col10/1731552218872/Put/seqid=0 2024-11-14T02:43:39,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741855_1031 (size=16931) 2024-11-14T02:43:39,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/4fa7ed0a3da64386aaf019be8fa167c5 2024-11-14T02:43:39,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/871ed68db2924844b7c968888219fc16 is 50, key is test_row_0/B:col10/1731552218872/Put/seqid=0 2024-11-14T02:43:39,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741856_1032 (size=12151) 2024-11-14T02:43:39,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/871ed68db2924844b7c968888219fc16 2024-11-14T02:43:39,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731552219928, max timestamp=1731552219977Num uniques 859; 2024-11-14T02:43:39,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T02:43:39,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552219928, max timestamp=1731552219977Num uniques 859; 2024-11-14T02:43:39,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:43:39,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552219928, max timestamp=1731552219977Num uniques 859; 2024-11-14T02:43:39,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:43:39,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/2c98dc7864554a689ec2de8b7e2ffd24 is 50, key is test_row_0/C:col10/1731552218872/Put/seqid=0 2024-11-14T02:43:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T02:43:40,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741857_1033 (size=12151) 2024-11-14T02:43:40,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/2c98dc7864554a689ec2de8b7e2ffd24 2024-11-14T02:43:40,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/4fa7ed0a3da64386aaf019be8fa167c5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4fa7ed0a3da64386aaf019be8fa167c5 2024-11-14T02:43:40,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4fa7ed0a3da64386aaf019be8fa167c5, entries=250, sequenceid=182, filesize=16.5 K 2024-11-14T02:43:40,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/871ed68db2924844b7c968888219fc16 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/871ed68db2924844b7c968888219fc16 2024-11-14T02:43:40,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/871ed68db2924844b7c968888219fc16, entries=150, sequenceid=182, filesize=11.9 K 2024-11-14T02:43:40,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/2c98dc7864554a689ec2de8b7e2ffd24 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/2c98dc7864554a689ec2de8b7e2ffd24 2024-11-14T02:43:40,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T02:43:40,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:40,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:40,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:40,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/2c98dc7864554a689ec2de8b7e2ffd24, entries=150, sequenceid=182, filesize=11.9 K 2024-11-14T02:43:40,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 167ms, sequenceid=182, compaction requested=true 2024-11-14T02:43:40,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:40,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:40,068 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:40,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:40,068 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:40,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:40,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:40,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:40,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:40,070 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:40,070 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:40,070 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/A is initiating minor compaction (all files) 2024-11-14T02:43:40,070 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/B is initiating minor compaction (all files) 2024-11-14T02:43:40,071 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/B in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:40,071 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/A in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:40,071 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/9c5c5b87768f4803afa791b63bb835b5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7028ef20e4374831a2a6bc98569212da, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/871ed68db2924844b7c968888219fc16] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=35.6 K 2024-11-14T02:43:40,071 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/2c0a765d035047828064ab46a8987035, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/2cd289e7e9834a15a94fd402d3ee0703, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4fa7ed0a3da64386aaf019be8fa167c5] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=40.3 K 2024-11-14T02:43:40,072 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c0a765d035047828064ab46a8987035, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731552215980 2024-11-14T02:43:40,072 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c5c5b87768f4803afa791b63bb835b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731552215980 2024-11-14T02:43:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:40,073 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 7028ef20e4374831a2a6bc98569212da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731552217630 2024-11-14T02:43:40,073 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cd289e7e9834a15a94fd402d3ee0703, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731552217630 2024-11-14T02:43:40,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-14T02:43:40,074 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 871ed68db2924844b7c968888219fc16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731552218872 2024-11-14T02:43:40,074 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fa7ed0a3da64386aaf019be8fa167c5, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731552218858 2024-11-14T02:43:40,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731552220022, max timestamp=1731552220072Num uniques 1018; 2024-11-14T02:43:40,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731552220022, max timestamp=1731552220072Num uniques 1168; 2024-11-14T02:43:40,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T02:43:40,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-14T02:43:40,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731552220022, max timestamp=1731552220072Num uniques 1018; 2024-11-14T02:43:40,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T02:43:40,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:40,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:40,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:40,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:40,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:40,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:40,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/c2db6a08cf7a4a43ad7885b71704e73b is 50, key is test_row_0/A:col10/1731552220072/Put/seqid=0 2024-11-14T02:43:40,109 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#B#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:40,117 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#A#compaction#23 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:40,118 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/a0639cdd245a4f65a6393a775e23db96 is 50, key is test_row_0/A:col10/1731552218872/Put/seqid=0 2024-11-14T02:43:40,119 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/ae0f169c8d604300aec5dcaf18d3b63d is 50, key is test_row_0/B:col10/1731552218872/Put/seqid=0 2024-11-14T02:43:40,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741858_1034 (size=16931) 2024-11-14T02:43:40,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741859_1035 (size=12391) 2024-11-14T02:43:40,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741860_1036 (size=12391) 2024-11-14T02:43:40,172 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/ae0f169c8d604300aec5dcaf18d3b63d as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/ae0f169c8d604300aec5dcaf18d3b63d 2024-11-14T02:43:40,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552220157, max timestamp=1731552220167Num uniques 859; 2024-11-14T02:43:40,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:43:40,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552220157, max timestamp=1731552220171Num uniques 859; 2024-11-14T02:43:40,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:43:40,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552220157, max timestamp=1731552220171Num uniques 859; 2024-11-14T02:43:40,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:43:40,186 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/B of b0f2fe4f29c62d92b3ee0b2d165c19cb into ae0f169c8d604300aec5dcaf18d3b63d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:40,186 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:40,186 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/B, priority=13, startTime=1731552220068; duration=0sec 2024-11-14T02:43:40,186 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:40,186 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:B 2024-11-14T02:43:40,186 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:40,188 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:40,188 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/C is initiating minor compaction (all files) 2024-11-14T02:43:40,188 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/C in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:40,189 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/7c79168ab0a44be1bbca12544587daf8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3afcd246c260451a90f7b08e98309f71, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/2c98dc7864554a689ec2de8b7e2ffd24] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=35.6 K 2024-11-14T02:43:40,189 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c79168ab0a44be1bbca12544587daf8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731552215980 2024-11-14T02:43:40,190 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 3afcd246c260451a90f7b08e98309f71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731552217630 2024-11-14T02:43:40,190 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c98dc7864554a689ec2de8b7e2ffd24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731552218872 2024-11-14T02:43:40,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T02:43:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T02:43:40,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:40,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:40,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:40,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,232 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:40,233 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/6770abe80c184b6bb5ee525d17e38e9b is 50, key is test_row_0/C:col10/1731552218872/Put/seqid=0 2024-11-14T02:43:40,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731552220206, max timestamp=1731552220246Num uniques 1018; 2024-11-14T02:43:40,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-14T02:43:40,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731552220206, max timestamp=1731552220246Num uniques 1018; 2024-11-14T02:43:40,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T02:43:40,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731552220206, max timestamp=1731552220246Num uniques 1018; 2024-11-14T02:43:40,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T02:43:40,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741861_1037 (size=12391) 2024-11-14T02:43:40,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552280288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552280290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10382, totalHeapSize=23848, min timestamp=1731552220261, max timestamp=1731552220292Num uniques 1177; 2024-11-14T02:43:40,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10389, totalHeapSize=23848, min timestamp=1731552220261, max timestamp=1731552220292Num uniques 1077; 2024-11-14T02:43:40,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10382, totalHeapSize=23848, min timestamp=1731552220261, max timestamp=1731552220292Num uniques 1077; 2024-11-14T02:43:40,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53892, new segment data size=10382, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-14T02:43:40,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49312, new segment data size=10382, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T02:43:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49319, new segment data size=10389, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T02:43:40,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731552220345, max timestamp=1731552220364Num uniques 1086; 2024-11-14T02:43:40,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-14T02:43:40,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731552220345, max timestamp=1731552220364Num uniques 1086; 2024-11-14T02:43:40,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-14T02:43:40,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731552220345, max timestamp=1731552220364Num uniques 1086; 2024-11-14T02:43:40,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-14T02:43:40,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T02:43:40,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:40,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:40,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:40,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1731552220388, max timestamp=1731552220422Num uniques 1095; 2024-11-14T02:43:40,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=18090, suffix heap size=141848,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=395 2024-11-14T02:43:40,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1731552220388, max timestamp=1731552220422Num uniques 1095; 2024-11-14T02:43:40,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=18090, suffix heap size=141848,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=395 2024-11-14T02:43:40,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1731552220388, max timestamp=1731552220422Num uniques 1095; 2024-11-14T02:43:40,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=18090, suffix heap size=141848,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=395 2024-11-14T02:43:40,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552280474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552280477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552280479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552280484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T02:43:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:40,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T02:43:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/c2db6a08cf7a4a43ad7885b71704e73b 2024-11-14T02:43:40,580 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/a0639cdd245a4f65a6393a775e23db96 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a0639cdd245a4f65a6393a775e23db96 2024-11-14T02:43:40,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/efdbf7535667489cbe640f7bb533f519 is 50, key is test_row_0/B:col10/1731552220072/Put/seqid=0 2024-11-14T02:43:40,593 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/A of b0f2fe4f29c62d92b3ee0b2d165c19cb into a0639cdd245a4f65a6393a775e23db96(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:40,593 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:40,594 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/A, priority=13, startTime=1731552220068; duration=0sec 2024-11-14T02:43:40,594 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:40,594 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:A 2024-11-14T02:43:40,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552280595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552280596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552280596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552280596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741862_1038 (size=12151) 2024-11-14T02:43:40,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/efdbf7535667489cbe640f7bb533f519 2024-11-14T02:43:40,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/194c39adf43e4ba4969050e3d78fd9d5 is 50, key is test_row_0/C:col10/1731552220072/Put/seqid=0 2024-11-14T02:43:40,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T02:43:40,682 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/6770abe80c184b6bb5ee525d17e38e9b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6770abe80c184b6bb5ee525d17e38e9b 2024-11-14T02:43:40,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:40,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:40,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:40,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,693 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/C of b0f2fe4f29c62d92b3ee0b2d165c19cb into 6770abe80c184b6bb5ee525d17e38e9b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:40,694 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:40,694 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/C, priority=13, startTime=1731552220068; duration=0sec 2024-11-14T02:43:40,694 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:40,694 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:C 2024-11-14T02:43:40,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741863_1039 (size=12151) 2024-11-14T02:43:40,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/194c39adf43e4ba4969050e3d78fd9d5 2024-11-14T02:43:40,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/c2db6a08cf7a4a43ad7885b71704e73b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/c2db6a08cf7a4a43ad7885b71704e73b 2024-11-14T02:43:40,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/c2db6a08cf7a4a43ad7885b71704e73b, entries=250, sequenceid=220, filesize=16.5 K 2024-11-14T02:43:40,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/efdbf7535667489cbe640f7bb533f519 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/efdbf7535667489cbe640f7bb533f519 2024-11-14T02:43:40,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/efdbf7535667489cbe640f7bb533f519, entries=150, sequenceid=220, filesize=11.9 K 2024-11-14T02:43:40,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/194c39adf43e4ba4969050e3d78fd9d5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/194c39adf43e4ba4969050e3d78fd9d5 2024-11-14T02:43:40,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/194c39adf43e4ba4969050e3d78fd9d5, entries=150, sequenceid=220, filesize=11.9 K 2024-11-14T02:43:40,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=161.02 KB/164880 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 691ms, sequenceid=220, compaction requested=false 2024-11-14T02:43:40,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:40,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=167.72 KB heapSize=462.47 KB 2024-11-14T02:43:40,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:40,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:40,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:40,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:40,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:40,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:40,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/a753feb7abc84ade8592763308b13577 is 50, key is test_row_0/A:col10/1731552220816/Put/seqid=0 2024-11-14T02:43:40,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T02:43:40,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:40,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:40,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:40,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:40,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552280826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552280846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552280854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552280855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741864_1040 (size=14741) 2024-11-14T02:43:40,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552280966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552280966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552280966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:40,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552280976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:41,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T02:43:41,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:41,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:41,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:41,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:41,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:41,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T02:43:41,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T02:43:41,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:41,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:41,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:41,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:41,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:41,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552281176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:41,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:41,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552281176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:41,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:41,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552281176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:41,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552281186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:41,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/a753feb7abc84ade8592763308b13577 2024-11-14T02:43:41,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T02:43:41,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:41,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:41,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:41,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:41,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:41,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/3ce1067af9c44f158cda85d70afdf288 is 50, key is test_row_0/B:col10/1731552220816/Put/seqid=0 2024-11-14T02:43:41,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741865_1041 (size=12301) 2024-11-14T02:43:41,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/3ce1067af9c44f158cda85d70afdf288 2024-11-14T02:43:41,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/0fecab8a02284e948263ce8e73067633 is 50, key is test_row_0/C:col10/1731552220816/Put/seqid=0 2024-11-14T02:43:41,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741866_1042 (size=12301) 2024-11-14T02:43:41,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/0fecab8a02284e948263ce8e73067633 2024-11-14T02:43:41,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/a753feb7abc84ade8592763308b13577 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a753feb7abc84ade8592763308b13577 2024-11-14T02:43:41,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a753feb7abc84ade8592763308b13577, entries=200, sequenceid=329, filesize=14.4 K 2024-11-14T02:43:41,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/3ce1067af9c44f158cda85d70afdf288 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/3ce1067af9c44f158cda85d70afdf288 2024-11-14T02:43:41,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/3ce1067af9c44f158cda85d70afdf288, entries=150, sequenceid=329, filesize=12.0 K 2024-11-14T02:43:41,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/0fecab8a02284e948263ce8e73067633 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/0fecab8a02284e948263ce8e73067633 2024-11-14T02:43:41,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/0fecab8a02284e948263ce8e73067633, entries=150, sequenceid=329, filesize=12.0 K 2024-11-14T02:43:41,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~462.42 KB/473520, currentSize=26.84 KB/27480 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 600ms, sequenceid=329, compaction requested=true 2024-11-14T02:43:41,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:41,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:41,417 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:41,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:41,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:41,418 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:41,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:41,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:41,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:41,425 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:41,425 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44063 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:41,425 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/A is initiating minor compaction (all files) 2024-11-14T02:43:41,425 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/B is initiating minor compaction (all files) 2024-11-14T02:43:41,425 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/A in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:41,425 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/B in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:41,426 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a0639cdd245a4f65a6393a775e23db96, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/c2db6a08cf7a4a43ad7885b71704e73b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a753feb7abc84ade8592763308b13577] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=43.0 K 2024-11-14T02:43:41,426 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/ae0f169c8d604300aec5dcaf18d3b63d, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/efdbf7535667489cbe640f7bb533f519, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/3ce1067af9c44f158cda85d70afdf288] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=36.0 K 2024-11-14T02:43:41,427 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting ae0f169c8d604300aec5dcaf18d3b63d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731552218872 2024-11-14T02:43:41,427 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0639cdd245a4f65a6393a775e23db96, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731552218872 2024-11-14T02:43:41,428 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting efdbf7535667489cbe640f7bb533f519, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731552220056 2024-11-14T02:43:41,428 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2db6a08cf7a4a43ad7885b71704e73b, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731552220022 2024-11-14T02:43:41,428 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting a753feb7abc84ade8592763308b13577, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731552220466 2024-11-14T02:43:41,429 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ce1067af9c44f158cda85d70afdf288, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731552220476 2024-11-14T02:43:41,452 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#A#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:41,452 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#B#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:41,453 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/57b88ad03d904fe08ff2f485473653b5 is 50, key is test_row_0/A:col10/1731552220816/Put/seqid=0 2024-11-14T02:43:41,453 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/5e9735171df045f3a9bcdda9a1dfb1ac is 50, key is test_row_0/B:col10/1731552220816/Put/seqid=0 2024-11-14T02:43:41,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T02:43:41,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:41,471 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-14T02:43:41,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:41,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:41,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:41,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:41,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:41,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:41,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741867_1043 (size=12643) 2024-11-14T02:43:41,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:41,514 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/5e9735171df045f3a9bcdda9a1dfb1ac as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/5e9735171df045f3a9bcdda9a1dfb1ac 2024-11-14T02:43:41,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/a36fa4a8a6bd40bab6a2bb889493f3f7 is 50, key is test_row_0/A:col10/1731552220825/Put/seqid=0 2024-11-14T02:43:41,547 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/B of b0f2fe4f29c62d92b3ee0b2d165c19cb into 5e9735171df045f3a9bcdda9a1dfb1ac(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:41,547 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:41,547 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/B, priority=13, startTime=1731552221418; duration=0sec 2024-11-14T02:43:41,548 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:41,548 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:B 2024-11-14T02:43:41,548 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:41,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741868_1044 (size=12643) 2024-11-14T02:43:41,551 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:41,551 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/C is initiating minor compaction (all files) 2024-11-14T02:43:41,551 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/C in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:41,552 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6770abe80c184b6bb5ee525d17e38e9b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/194c39adf43e4ba4969050e3d78fd9d5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/0fecab8a02284e948263ce8e73067633] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=36.0 K 2024-11-14T02:43:41,552 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 6770abe80c184b6bb5ee525d17e38e9b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731552218872 2024-11-14T02:43:41,554 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 194c39adf43e4ba4969050e3d78fd9d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731552220056 2024-11-14T02:43:41,556 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fecab8a02284e948263ce8e73067633, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731552220476 2024-11-14T02:43:41,560 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/57b88ad03d904fe08ff2f485473653b5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/57b88ad03d904fe08ff2f485473653b5 2024-11-14T02:43:41,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731552221529, max timestamp=1731552221559Num uniques 859; 2024-11-14T02:43:41,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T02:43:41,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731552221529, max timestamp=1731552221559Num uniques 859; 2024-11-14T02:43:41,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T02:43:41,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552221536, max timestamp=1731552221559Num uniques 859; 2024-11-14T02:43:41,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:43:41,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741869_1045 (size=9857) 2024-11-14T02:43:41,576 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:41,577 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/596226adb2f44920be3826732900c0d1 is 50, key is test_row_0/C:col10/1731552220816/Put/seqid=0 2024-11-14T02:43:41,579 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/A of b0f2fe4f29c62d92b3ee0b2d165c19cb into 57b88ad03d904fe08ff2f485473653b5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:41,579 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:41,579 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/A, priority=13, startTime=1731552221417; duration=0sec 2024-11-14T02:43:41,579 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:41,579 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:A 2024-11-14T02:43:41,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741870_1046 (size=12643) 2024-11-14T02:43:41,612 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/596226adb2f44920be3826732900c0d1 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/596226adb2f44920be3826732900c0d1 2024-11-14T02:43:41,624 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/C of b0f2fe4f29c62d92b3ee0b2d165c19cb into 596226adb2f44920be3826732900c0d1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:41,624 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:41,625 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/C, priority=13, startTime=1731552221418; duration=0sec 2024-11-14T02:43:41,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731552221588, max timestamp=1731552221620Num uniques 1168; 2024-11-14T02:43:41,625 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:41,625 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:C 2024-11-14T02:43:41,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=16848, suffix heap size=149440,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=368 2024-11-14T02:43:41,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731552221588, max timestamp=1731552221620Num uniques 1168; 2024-11-14T02:43:41,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=16848, suffix heap size=149440,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=368 2024-11-14T02:43:41,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731552221588, max timestamp=1731552221620Num uniques 1018; 2024-11-14T02:43:41,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=16848, suffix heap size=133840,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=368 2024-11-14T02:43:41,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552281660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:41,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552281666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:41,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:41,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552281671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552281672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:41,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552281776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552281776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:41,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552281785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552281785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:41,975 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/a36fa4a8a6bd40bab6a2bb889493f3f7 2024-11-14T02:43:41,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552281986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:41,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:41,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552281988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:41,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552281995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:42,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:42,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552281995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:42,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:42,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/07c5ef1eb8cc4a359a13f1db5aa688b3 is 50, key is test_row_0/B:col10/1731552220825/Put/seqid=0 2024-11-14T02:43:42,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741871_1047 (size=9857) 2024-11-14T02:43:42,023 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/07c5ef1eb8cc4a359a13f1db5aa688b3 2024-11-14T02:43:42,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/10bc312a8fe9457f995d4673729596f5 is 50, key is test_row_0/C:col10/1731552220825/Put/seqid=0 2024-11-14T02:43:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T02:43:42,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741872_1048 (size=9857) 2024-11-14T02:43:42,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731552282246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:42,247 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in b0f2fe4f29c62d92b3ee0b2d165c19cb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8281 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:43:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:42,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:42,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552282295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:42,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:42,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552282306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552282305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:42,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:42,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552282306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:42,463 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/10bc312a8fe9457f995d4673729596f5 2024-11-14T02:43:42,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/a36fa4a8a6bd40bab6a2bb889493f3f7 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a36fa4a8a6bd40bab6a2bb889493f3f7 2024-11-14T02:43:42,499 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a36fa4a8a6bd40bab6a2bb889493f3f7, entries=100, sequenceid=336, filesize=9.6 K 2024-11-14T02:43:42,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/07c5ef1eb8cc4a359a13f1db5aa688b3 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/07c5ef1eb8cc4a359a13f1db5aa688b3 2024-11-14T02:43:42,514 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/07c5ef1eb8cc4a359a13f1db5aa688b3, entries=100, sequenceid=336, filesize=9.6 K 2024-11-14T02:43:42,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/10bc312a8fe9457f995d4673729596f5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/10bc312a8fe9457f995d4673729596f5 2024-11-14T02:43:42,544 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/10bc312a8fe9457f995d4673729596f5, entries=100, sequenceid=336, filesize=9.6 K 2024-11-14T02:43:42,546 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=161.02 KB/164880 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 1075ms, sequenceid=336, compaction requested=false 2024-11-14T02:43:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-14T02:43:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-14T02:43:42,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-14T02:43:42,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6490 sec 2024-11-14T02:43:42,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.6690 sec 2024-11-14T02:43:42,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731552221651, max timestamp=1731552222814Num uniques 1227; 2024-11-14T02:43:42,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=56192, new segment data size=8102, suffix heap size=155576,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1227, new segment cells count=177 2024-11-14T02:43:42,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731552221651, max timestamp=1731552222814Num uniques 1227; 2024-11-14T02:43:42,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=120.76 KB heapSize=331.98 KB 2024-11-14T02:43:42,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=56192, new segment data size=8102, suffix heap size=155576,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1227, new segment cells count=177 2024-11-14T02:43:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:42,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:42,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:42,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T02:43:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:42,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/a74556c26f5d469b812bad2895e81395 is 50, key is test_row_0/A:col10/1731552222814/Put/seqid=0 2024-11-14T02:43:42,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741873_1049 (size=14741) 2024-11-14T02:43:42,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/a74556c26f5d469b812bad2895e81395 2024-11-14T02:43:42,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552282895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552282896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:42,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:42,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552282898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:42,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552282899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:42,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/7053a56c6bc54d74847452b7b4a5c1cc is 50, key is test_row_0/B:col10/1731552222814/Put/seqid=0 2024-11-14T02:43:42,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741874_1050 (size=12301) 2024-11-14T02:43:42,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/7053a56c6bc54d74847452b7b4a5c1cc 2024-11-14T02:43:42,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/0fe4eb51683048e1a82ea848080e9cf2 is 50, key is test_row_0/C:col10/1731552222814/Put/seqid=0 2024-11-14T02:43:42,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741875_1051 (size=12301) 2024-11-14T02:43:42,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/0fe4eb51683048e1a82ea848080e9cf2 2024-11-14T02:43:42,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/a74556c26f5d469b812bad2895e81395 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a74556c26f5d469b812bad2895e81395 2024-11-14T02:43:43,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552283005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552283005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552283006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552283006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a74556c26f5d469b812bad2895e81395, entries=200, sequenceid=394, filesize=14.4 K 2024-11-14T02:43:43,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/7053a56c6bc54d74847452b7b4a5c1cc as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7053a56c6bc54d74847452b7b4a5c1cc 2024-11-14T02:43:43,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7053a56c6bc54d74847452b7b4a5c1cc, entries=150, sequenceid=394, filesize=12.0 K 2024-11-14T02:43:43,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/0fe4eb51683048e1a82ea848080e9cf2 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/0fe4eb51683048e1a82ea848080e9cf2 2024-11-14T02:43:43,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/0fe4eb51683048e1a82ea848080e9cf2, entries=150, sequenceid=394, filesize=12.0 K 2024-11-14T02:43:43,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~198.22 KB/202976, currentSize=114.05 KB/116790 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 233ms, sequenceid=394, compaction requested=true 2024-11-14T02:43:43,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:43,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:43,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:43,056 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:43,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:43,057 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:43,058 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34801 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:43,058 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37241 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:43,058 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/A is initiating minor compaction (all files) 2024-11-14T02:43:43,058 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/B is initiating minor compaction (all files) 2024-11-14T02:43:43,058 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/A in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:43,058 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/B in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:43,059 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/57b88ad03d904fe08ff2f485473653b5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a36fa4a8a6bd40bab6a2bb889493f3f7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a74556c26f5d469b812bad2895e81395] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=36.4 K 2024-11-14T02:43:43,059 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/5e9735171df045f3a9bcdda9a1dfb1ac, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/07c5ef1eb8cc4a359a13f1db5aa688b3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7053a56c6bc54d74847452b7b4a5c1cc] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=34.0 K 2024-11-14T02:43:43,063 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e9735171df045f3a9bcdda9a1dfb1ac, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731552220476 2024-11-14T02:43:43,063 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57b88ad03d904fe08ff2f485473653b5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731552220476 2024-11-14T02:43:43,063 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting a36fa4a8a6bd40bab6a2bb889493f3f7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731552220825 2024-11-14T02:43:43,064 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting a74556c26f5d469b812bad2895e81395, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731552221651 2024-11-14T02:43:43,065 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 07c5ef1eb8cc4a359a13f1db5aa688b3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731552220825 2024-11-14T02:43:43,065 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 7053a56c6bc54d74847452b7b4a5c1cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731552221663 2024-11-14T02:43:43,083 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:43,084 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:43,084 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/4b6fab1f9bf44db996e308e08c21637d is 50, key is test_row_0/A:col10/1731552222814/Put/seqid=0 2024-11-14T02:43:43,084 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/c76cd0eec6654d3f8127c4fe78133788 is 50, key is test_row_0/B:col10/1731552222814/Put/seqid=0 2024-11-14T02:43:43,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741877_1053 (size=12745) 2024-11-14T02:43:43,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741876_1052 (size=12745) 2024-11-14T02:43:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:43,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T02:43:43,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:43,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:43,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:43,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:43,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:43,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:43,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/5413a6c151074dbc9af41d43202841a3 is 50, key is test_row_0/A:col10/1731552222892/Put/seqid=0 2024-11-14T02:43:43,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552283249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552283252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552283254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552283251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741878_1054 (size=12301) 2024-11-14T02:43:43,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/5413a6c151074dbc9af41d43202841a3 2024-11-14T02:43:43,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/c7d33a94c5984d3a8194d7989cf5adce is 50, key is test_row_0/B:col10/1731552222892/Put/seqid=0 2024-11-14T02:43:43,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741879_1055 (size=12301) 2024-11-14T02:43:43,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/c7d33a94c5984d3a8194d7989cf5adce 2024-11-14T02:43:43,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/b7271eb02cf94c32a5dd57026abf6bc8 is 50, key is test_row_0/C:col10/1731552222892/Put/seqid=0 2024-11-14T02:43:43,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552283366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552283366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552283366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552283366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741880_1056 (size=12301) 2024-11-14T02:43:43,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/b7271eb02cf94c32a5dd57026abf6bc8 2024-11-14T02:43:43,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/5413a6c151074dbc9af41d43202841a3 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/5413a6c151074dbc9af41d43202841a3 2024-11-14T02:43:43,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/5413a6c151074dbc9af41d43202841a3, entries=150, sequenceid=415, filesize=12.0 K 2024-11-14T02:43:43,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/c7d33a94c5984d3a8194d7989cf5adce as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c7d33a94c5984d3a8194d7989cf5adce 2024-11-14T02:43:43,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c7d33a94c5984d3a8194d7989cf5adce, entries=150, sequenceid=415, filesize=12.0 K 2024-11-14T02:43:43,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/b7271eb02cf94c32a5dd57026abf6bc8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/b7271eb02cf94c32a5dd57026abf6bc8 2024-11-14T02:43:43,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/b7271eb02cf94c32a5dd57026abf6bc8, entries=150, sequenceid=415, filesize=12.0 K 2024-11-14T02:43:43,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 204ms, sequenceid=415, compaction requested=true 2024-11-14T02:43:43,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:43,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:A, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:43,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T02:43:43,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:B, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:43,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T02:43:43,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:43,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-14T02:43:43,550 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/c76cd0eec6654d3f8127c4fe78133788 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c76cd0eec6654d3f8127c4fe78133788 2024-11-14T02:43:43,551 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/4b6fab1f9bf44db996e308e08c21637d as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4b6fab1f9bf44db996e308e08c21637d 2024-11-14T02:43:43,564 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/A of b0f2fe4f29c62d92b3ee0b2d165c19cb into 4b6fab1f9bf44db996e308e08c21637d(size=12.4 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:43,564 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:43,564 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/A, priority=13, startTime=1731552223056; duration=0sec 2024-11-14T02:43:43,565 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-14T02:43:43,565 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:A 2024-11-14T02:43:43,565 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:A 2024-11-14T02:43:43,565 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:43,568 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47102 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:43,568 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/C is initiating minor compaction (all files) 2024-11-14T02:43:43,568 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/C in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:43,569 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/596226adb2f44920be3826732900c0d1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/10bc312a8fe9457f995d4673729596f5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/0fe4eb51683048e1a82ea848080e9cf2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/b7271eb02cf94c32a5dd57026abf6bc8] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=46.0 K 2024-11-14T02:43:43,571 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/B of b0f2fe4f29c62d92b3ee0b2d165c19cb into c76cd0eec6654d3f8127c4fe78133788(size=12.4 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:43,571 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:43,571 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/B, priority=13, startTime=1731552223056; duration=0sec 2024-11-14T02:43:43,571 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T02:43:43,571 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:B 2024-11-14T02:43:43,572 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:B 2024-11-14T02:43:43,572 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-14T02:43:43,572 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T02:43:43,572 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T02:43:43,573 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. because compaction request was cancelled 2024-11-14T02:43:43,573 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:C 2024-11-14T02:43:43,573 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T02:43:43,574 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T02:43:43,574 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T02:43:43,574 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. because compaction request was cancelled 2024-11-14T02:43:43,575 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:B 2024-11-14T02:43:43,575 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T02:43:43,575 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 596226adb2f44920be3826732900c0d1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731552220476 2024-11-14T02:43:43,577 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10bc312a8fe9457f995d4673729596f5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731552220825 2024-11-14T02:43:43,577 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T02:43:43,577 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T02:43:43,577 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. because compaction request was cancelled 2024-11-14T02:43:43,578 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:A 2024-11-14T02:43:43,578 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fe4eb51683048e1a82ea848080e9cf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731552221663 2024-11-14T02:43:43,578 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7271eb02cf94c32a5dd57026abf6bc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731552222892 2024-11-14T02:43:43,603 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:43,604 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/c2d0eeab8d544d3aa718054f91f3af45 is 50, key is test_row_0/C:col10/1731552222892/Put/seqid=0 2024-11-14T02:43:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:43,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T02:43:43,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:43,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:43,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:43,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:43,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:43,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:43,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/bc8821673a674cc59e93b32542dc5c84 is 50, key is test_row_0/A:col10/1731552223605/Put/seqid=0 2024-11-14T02:43:43,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741881_1057 (size=12779) 2024-11-14T02:43:43,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741882_1058 (size=12301) 2024-11-14T02:43:43,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/bc8821673a674cc59e93b32542dc5c84 2024-11-14T02:43:43,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/52d34335ac5e4e5db56b03b6f89938e0 is 50, key is test_row_0/B:col10/1731552223605/Put/seqid=0 2024-11-14T02:43:43,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741883_1059 (size=12301) 2024-11-14T02:43:43,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552283702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552283745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552283745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552283750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552283855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552283865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552283865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552283865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,070 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/c2d0eeab8d544d3aa718054f91f3af45 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/c2d0eeab8d544d3aa718054f91f3af45 2024-11-14T02:43:44,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552284066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T02:43:44,076 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:44,077 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-14T02:43:44,081 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:44,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552284075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T02:43:44,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552284075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,089 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:44,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:44,095 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/C of b0f2fe4f29c62d92b3ee0b2d165c19cb into c2d0eeab8d544d3aa718054f91f3af45(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:44,095 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:44,095 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/C, priority=12, startTime=1731552223424; duration=0sec 2024-11-14T02:43:44,095 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:44,095 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:C 2024-11-14T02:43:44,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552284088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/52d34335ac5e4e5db56b03b6f89938e0 2024-11-14T02:43:44,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/3b7ce1c1931c4651b01325f5f3dc7e4b is 50, key is test_row_0/C:col10/1731552223605/Put/seqid=0 2024-11-14T02:43:44,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741884_1060 (size=12301) 2024-11-14T02:43:44,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/3b7ce1c1931c4651b01325f5f3dc7e4b 2024-11-14T02:43:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T02:43:44,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/bc8821673a674cc59e93b32542dc5c84 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/bc8821673a674cc59e93b32542dc5c84 2024-11-14T02:43:44,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/bc8821673a674cc59e93b32542dc5c84, entries=150, sequenceid=431, filesize=12.0 K 2024-11-14T02:43:44,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/52d34335ac5e4e5db56b03b6f89938e0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/52d34335ac5e4e5db56b03b6f89938e0 2024-11-14T02:43:44,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/52d34335ac5e4e5db56b03b6f89938e0, entries=150, sequenceid=431, filesize=12.0 K 2024-11-14T02:43:44,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/3b7ce1c1931c4651b01325f5f3dc7e4b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3b7ce1c1931c4651b01325f5f3dc7e4b 2024-11-14T02:43:44,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3b7ce1c1931c4651b01325f5f3dc7e4b, entries=150, sequenceid=431, filesize=12.0 K 2024-11-14T02:43:44,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 617ms, sequenceid=431, compaction requested=true 2024-11-14T02:43:44,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:44,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:44,226 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:44,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:44,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:44,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:44,226 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:44,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:44,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:44,227 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:44,228 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/A is initiating minor compaction (all files) 2024-11-14T02:43:44,228 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:44,228 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/A in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:44,228 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/B is initiating minor compaction (all files) 2024-11-14T02:43:44,228 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/B in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:44,228 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4b6fab1f9bf44db996e308e08c21637d, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/5413a6c151074dbc9af41d43202841a3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/bc8821673a674cc59e93b32542dc5c84] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=36.5 K 2024-11-14T02:43:44,228 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c76cd0eec6654d3f8127c4fe78133788, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c7d33a94c5984d3a8194d7989cf5adce, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/52d34335ac5e4e5db56b03b6f89938e0] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=36.5 K 2024-11-14T02:43:44,229 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting c76cd0eec6654d3f8127c4fe78133788, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731552221663 2024-11-14T02:43:44,229 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b6fab1f9bf44db996e308e08c21637d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731552221663 2024-11-14T02:43:44,229 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5413a6c151074dbc9af41d43202841a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731552222892 2024-11-14T02:43:44,229 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting c7d33a94c5984d3a8194d7989cf5adce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731552222892 2024-11-14T02:43:44,230 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 52d34335ac5e4e5db56b03b6f89938e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731552223605 2024-11-14T02:43:44,230 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc8821673a674cc59e93b32542dc5c84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731552223605 2024-11-14T02:43:44,243 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:44,244 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/d71350b894cf4815b1bcf7a317ab8507 is 50, key is test_row_0/B:col10/1731552223605/Put/seqid=0 2024-11-14T02:43:44,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T02:43:44,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:44,246 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T02:43:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:44,249 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#A#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:44,249 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/8f2fb4b3d79f45c0a215f0612e93b8c9 is 50, key is test_row_0/A:col10/1731552223605/Put/seqid=0 2024-11-14T02:43:44,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/11fa5577bb3944e584ae55b4d8b2311b is 50, key is test_row_0/A:col10/1731552223709/Put/seqid=0 2024-11-14T02:43:44,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741885_1061 (size=12847) 2024-11-14T02:43:44,294 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/d71350b894cf4815b1bcf7a317ab8507 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d71350b894cf4815b1bcf7a317ab8507 2024-11-14T02:43:44,305 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/B of b0f2fe4f29c62d92b3ee0b2d165c19cb into d71350b894cf4815b1bcf7a317ab8507(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:44,305 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:44,305 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/B, priority=13, startTime=1731552224226; duration=0sec 2024-11-14T02:43:44,305 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:44,305 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:B 2024-11-14T02:43:44,305 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T02:43:44,306 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T02:43:44,306 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T02:43:44,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741886_1062 (size=12847) 2024-11-14T02:43:44,309 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. because compaction request was cancelled 2024-11-14T02:43:44,309 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:C 2024-11-14T02:43:44,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741887_1063 (size=12301) 2024-11-14T02:43:44,343 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/11fa5577bb3944e584ae55b4d8b2311b 2024-11-14T02:43:44,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/d1e7b70eb0f34b40a9e0b453bbb1ae3e is 50, key is test_row_0/B:col10/1731552223709/Put/seqid=0 2024-11-14T02:43:44,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741888_1064 (size=12301) 2024-11-14T02:43:44,381 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/d1e7b70eb0f34b40a9e0b453bbb1ae3e 2024-11-14T02:43:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:44,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:44,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/cdc969c4ea8841cea16db4df75d90708 is 50, key is test_row_0/C:col10/1731552223709/Put/seqid=0 2024-11-14T02:43:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T02:43:44,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741889_1065 (size=12301) 2024-11-14T02:43:44,416 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/cdc969c4ea8841cea16db4df75d90708 2024-11-14T02:43:44,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/11fa5577bb3944e584ae55b4d8b2311b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/11fa5577bb3944e584ae55b4d8b2311b 2024-11-14T02:43:44,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552284431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,440 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/11fa5577bb3944e584ae55b4d8b2311b, entries=150, sequenceid=450, filesize=12.0 K 2024-11-14T02:43:44,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552284433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/d1e7b70eb0f34b40a9e0b453bbb1ae3e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d1e7b70eb0f34b40a9e0b453bbb1ae3e 2024-11-14T02:43:44,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552284438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552284438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,461 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d1e7b70eb0f34b40a9e0b453bbb1ae3e, entries=150, sequenceid=450, filesize=12.0 K 2024-11-14T02:43:44,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/cdc969c4ea8841cea16db4df75d90708 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/cdc969c4ea8841cea16db4df75d90708 2024-11-14T02:43:44,472 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/cdc969c4ea8841cea16db4df75d90708, entries=150, sequenceid=450, filesize=12.0 K 2024-11-14T02:43:44,474 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 229ms, sequenceid=450, compaction requested=true 2024-11-14T02:43:44,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:44,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:44,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-14T02:43:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-14T02:43:44,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T02:43:44,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 390 msec 2024-11-14T02:43:44,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 406 msec 2024-11-14T02:43:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:44,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T02:43:44,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:44,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:44,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:44,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:44,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:44,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:44,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/9e0e1e0da97d48e085a782141cc93484 is 50, key is test_row_0/A:col10/1731552224548/Put/seqid=0 2024-11-14T02:43:44,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741890_1066 (size=14741) 2024-11-14T02:43:44,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/9e0e1e0da97d48e085a782141cc93484 2024-11-14T02:43:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552284605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552284605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552284608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/fd05f3eefdb8414daf21b329bc82c2fc is 50, key is test_row_0/B:col10/1731552224548/Put/seqid=0 2024-11-14T02:43:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552284609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741891_1067 (size=12301) 2024-11-14T02:43:44,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/fd05f3eefdb8414daf21b329bc82c2fc 2024-11-14T02:43:44,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/114f30f356414f318a3bcc0cb7b34b78 is 50, key is test_row_0/C:col10/1731552224548/Put/seqid=0 2024-11-14T02:43:44,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741892_1068 (size=12301) 2024-11-14T02:43:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T02:43:44,706 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:44,708 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-14T02:43:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T02:43:44,711 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:44,712 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:44,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:44,718 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/8f2fb4b3d79f45c0a215f0612e93b8c9 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/8f2fb4b3d79f45c0a215f0612e93b8c9 2024-11-14T02:43:44,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552284719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,732 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/A of b0f2fe4f29c62d92b3ee0b2d165c19cb into 8f2fb4b3d79f45c0a215f0612e93b8c9(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:44,732 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:44,733 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/A, priority=13, startTime=1731552224226; duration=0sec 2024-11-14T02:43:44,733 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:44,733 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:A 2024-11-14T02:43:44,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552284726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552284726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552284724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T02:43:44,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T02:43:44,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:44,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:44,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:44,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:44,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:44,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552284925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552284945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552284946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:44,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552284946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T02:43:45,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T02:43:45,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:45,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:45,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:45,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:45,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:45,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:45,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/114f30f356414f318a3bcc0cb7b34b78 2024-11-14T02:43:45,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/9e0e1e0da97d48e085a782141cc93484 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/9e0e1e0da97d48e085a782141cc93484 2024-11-14T02:43:45,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/9e0e1e0da97d48e085a782141cc93484, entries=200, sequenceid=467, filesize=14.4 K 2024-11-14T02:43:45,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/fd05f3eefdb8414daf21b329bc82c2fc as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/fd05f3eefdb8414daf21b329bc82c2fc 2024-11-14T02:43:45,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/fd05f3eefdb8414daf21b329bc82c2fc, entries=150, sequenceid=467, filesize=12.0 K 2024-11-14T02:43:45,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/114f30f356414f318a3bcc0cb7b34b78 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/114f30f356414f318a3bcc0cb7b34b78 2024-11-14T02:43:45,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/114f30f356414f318a3bcc0cb7b34b78, entries=150, sequenceid=467, filesize=12.0 K 2024-11-14T02:43:45,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 600ms, sequenceid=467, compaction requested=true 2024-11-14T02:43:45,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:45,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:45,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:45,150 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:45,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:45,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:45,150 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:45,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:45,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:45,151 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:45,151 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:45,151 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/A is initiating minor compaction (all files) 2024-11-14T02:43:45,151 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/B is initiating minor compaction (all files) 2024-11-14T02:43:45,152 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/A in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:45,152 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/B in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:45,152 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/8f2fb4b3d79f45c0a215f0612e93b8c9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/11fa5577bb3944e584ae55b4d8b2311b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/9e0e1e0da97d48e085a782141cc93484] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=39.0 K 2024-11-14T02:43:45,152 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d71350b894cf4815b1bcf7a317ab8507, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d1e7b70eb0f34b40a9e0b453bbb1ae3e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/fd05f3eefdb8414daf21b329bc82c2fc] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=36.6 K 2024-11-14T02:43:45,153 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f2fb4b3d79f45c0a215f0612e93b8c9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731552223605 2024-11-14T02:43:45,153 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting d71350b894cf4815b1bcf7a317ab8507, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731552223605 2024-11-14T02:43:45,154 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11fa5577bb3944e584ae55b4d8b2311b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731552223697 2024-11-14T02:43:45,154 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting d1e7b70eb0f34b40a9e0b453bbb1ae3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731552223697 2024-11-14T02:43:45,155 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e0e1e0da97d48e085a782141cc93484, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731552224425 2024-11-14T02:43:45,156 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting fd05f3eefdb8414daf21b329bc82c2fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731552224425 2024-11-14T02:43:45,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T02:43:45,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:45,181 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T02:43:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:45,189 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#B#compaction#56 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:45,190 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/fba2ef20b40d45ec9d85fd5503a8f32c is 50, key is test_row_0/B:col10/1731552224548/Put/seqid=0 2024-11-14T02:43:45,207 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:45,209 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/7b20f5ac91c1463ba1282b1d755c9aa5 is 50, key is test_row_0/A:col10/1731552224548/Put/seqid=0 2024-11-14T02:43:45,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/5efa3f6a4e794154a30c785d9a2669e0 is 50, key is test_row_0/A:col10/1731552224604/Put/seqid=0 2024-11-14T02:43:45,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:45,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:45,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741894_1070 (size=12949) 2024-11-14T02:43:45,291 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/7b20f5ac91c1463ba1282b1d755c9aa5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/7b20f5ac91c1463ba1282b1d755c9aa5 2024-11-14T02:43:45,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552285282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552285285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552285286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552285288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741893_1069 (size=12949) 2024-11-14T02:43:45,312 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/A of b0f2fe4f29c62d92b3ee0b2d165c19cb into 7b20f5ac91c1463ba1282b1d755c9aa5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:45,313 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:45,313 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/A, priority=13, startTime=1731552225150; duration=0sec 2024-11-14T02:43:45,314 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:45,314 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:A 2024-11-14T02:43:45,314 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:45,316 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/fba2ef20b40d45ec9d85fd5503a8f32c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/fba2ef20b40d45ec9d85fd5503a8f32c 2024-11-14T02:43:45,318 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:45,319 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/C is initiating minor compaction (all files) 2024-11-14T02:43:45,319 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/C in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:45,320 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/c2d0eeab8d544d3aa718054f91f3af45, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3b7ce1c1931c4651b01325f5f3dc7e4b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/cdc969c4ea8841cea16db4df75d90708, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/114f30f356414f318a3bcc0cb7b34b78] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=48.5 K 2024-11-14T02:43:45,324 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2d0eeab8d544d3aa718054f91f3af45, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731552222892 2024-11-14T02:43:45,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741895_1071 (size=12301) 2024-11-14T02:43:45,327 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b7ce1c1931c4651b01325f5f3dc7e4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731552223605 2024-11-14T02:43:45,328 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdc969c4ea8841cea16db4df75d90708, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731552223697 2024-11-14T02:43:45,329 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 114f30f356414f318a3bcc0cb7b34b78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731552224425 2024-11-14T02:43:45,330 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/B of b0f2fe4f29c62d92b3ee0b2d165c19cb into fba2ef20b40d45ec9d85fd5503a8f32c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:45,330 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:45,332 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/B, priority=13, startTime=1731552225150; duration=0sec 2024-11-14T02:43:45,332 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:45,332 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:B 2024-11-14T02:43:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T02:43:45,349 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:45,350 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/6daf9839f57a44628c9c4c6b8a6181ff is 50, key is test_row_0/C:col10/1731552224548/Put/seqid=0 2024-11-14T02:43:45,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741896_1072 (size=12915) 2024-11-14T02:43:45,386 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/6daf9839f57a44628c9c4c6b8a6181ff as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6daf9839f57a44628c9c4c6b8a6181ff 2024-11-14T02:43:45,400 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/C of b0f2fe4f29c62d92b3ee0b2d165c19cb into 6daf9839f57a44628c9c4c6b8a6181ff(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:45,400 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:45,400 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/C, priority=12, startTime=1731552225150; duration=0sec 2024-11-14T02:43:45,400 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:45,400 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:C 2024-11-14T02:43:45,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552285406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552285406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552285406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552285406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552285625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552285625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552285625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552285625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,727 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/5efa3f6a4e794154a30c785d9a2669e0 2024-11-14T02:43:45,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/ab7f55473e7b4c43853ff2a9ff3505cd is 50, key is test_row_0/B:col10/1731552224604/Put/seqid=0 2024-11-14T02:43:45,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741897_1073 (size=12301) 2024-11-14T02:43:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T02:43:45,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552285935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552285935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552285936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552285936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:46,143 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/ab7f55473e7b4c43853ff2a9ff3505cd 2024-11-14T02:43:46,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/67b2ba534c6842d68c8fef456f275f55 is 50, key is test_row_0/C:col10/1731552224604/Put/seqid=0 2024-11-14T02:43:46,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741898_1074 (size=12301) 2024-11-14T02:43:46,164 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/67b2ba534c6842d68c8fef456f275f55 2024-11-14T02:43:46,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/5efa3f6a4e794154a30c785d9a2669e0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/5efa3f6a4e794154a30c785d9a2669e0 2024-11-14T02:43:46,178 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/5efa3f6a4e794154a30c785d9a2669e0, entries=150, sequenceid=485, filesize=12.0 K 2024-11-14T02:43:46,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/ab7f55473e7b4c43853ff2a9ff3505cd as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/ab7f55473e7b4c43853ff2a9ff3505cd 2024-11-14T02:43:46,185 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/ab7f55473e7b4c43853ff2a9ff3505cd, entries=150, sequenceid=485, filesize=12.0 K 2024-11-14T02:43:46,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/67b2ba534c6842d68c8fef456f275f55 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/67b2ba534c6842d68c8fef456f275f55 2024-11-14T02:43:46,195 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/67b2ba534c6842d68c8fef456f275f55, entries=150, sequenceid=485, filesize=12.0 K 2024-11-14T02:43:46,196 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 1016ms, sequenceid=485, compaction requested=false 2024-11-14T02:43:46,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:46,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:46,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-14T02:43:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-14T02:43:46,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-14T02:43:46,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4850 sec 2024-11-14T02:43:46,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.4920 sec 2024-11-14T02:43:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:46,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T02:43:46,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:46,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:46,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:46,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:46,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:46,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:46,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/adf5b4194a4f423d8c6d9f5b2ab80c28 is 50, key is test_row_0/A:col10/1731552226447/Put/seqid=0 2024-11-14T02:43:46,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741899_1075 (size=14741) 2024-11-14T02:43:46,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552286482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:46,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:46,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552286482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552286482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552286484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:46,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:46,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552286595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:46,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552286608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:46,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:46,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552286609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:46,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552286609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:46,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552286805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:46,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552286815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:46,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:46,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:46,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552286826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:46,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:46,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:46,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552286826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:46,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T02:43:46,855 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:46,856 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-14T02:43:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T02:43:46,858 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:46,858 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:46,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:46,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/adf5b4194a4f423d8c6d9f5b2ab80c28 2024-11-14T02:43:46,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/3e5f6655d58142f3bbdedc15a8df3f87 is 50, key is test_row_0/B:col10/1731552226447/Put/seqid=0 2024-11-14T02:43:46,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741900_1076 (size=12301) 2024-11-14T02:43:46,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T02:43:47,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T02:43:47,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:47,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:47,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:47,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552287115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:47,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:47,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552287125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:47,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:47,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552287135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552287136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:47,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T02:43:47,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:47,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:47,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:47,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T02:43:47,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/3e5f6655d58142f3bbdedc15a8df3f87 2024-11-14T02:43:47,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/af47ed6ebcbf496a9011ac8bd9c746ae is 50, key is test_row_0/C:col10/1731552226447/Put/seqid=0 2024-11-14T02:43:47,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741901_1077 (size=12301) 2024-11-14T02:43:47,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T02:43:47,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:47,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:47,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:47,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T02:43:47,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:47,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:47,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:47,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T02:43:47,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552287625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:47,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:47,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T02:43:47,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:47,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:47,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:47,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:47,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552287636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:47,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:47,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552287646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552287646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:47,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:47,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:47,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:47,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/af47ed6ebcbf496a9011ac8bd9c746ae 2024-11-14T02:43:47,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/adf5b4194a4f423d8c6d9f5b2ab80c28 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/adf5b4194a4f423d8c6d9f5b2ab80c28 2024-11-14T02:43:47,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/adf5b4194a4f423d8c6d9f5b2ab80c28, entries=200, sequenceid=504, filesize=14.4 K 2024-11-14T02:43:47,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/3e5f6655d58142f3bbdedc15a8df3f87 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/3e5f6655d58142f3bbdedc15a8df3f87 2024-11-14T02:43:47,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/3e5f6655d58142f3bbdedc15a8df3f87, entries=150, sequenceid=504, filesize=12.0 K 2024-11-14T02:43:47,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/af47ed6ebcbf496a9011ac8bd9c746ae as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/af47ed6ebcbf496a9011ac8bd9c746ae 2024-11-14T02:43:47,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/af47ed6ebcbf496a9011ac8bd9c746ae, entries=150, sequenceid=504, filesize=12.0 K 2024-11-14T02:43:47,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 1320ms, sequenceid=504, compaction requested=true 2024-11-14T02:43:47,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:47,768 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:47,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:47,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:47,769 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:47,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:47,770 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/A is initiating minor compaction (all files) 2024-11-14T02:43:47,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:47,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:47,770 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/A in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:47,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T02:43:47,770 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/7b20f5ac91c1463ba1282b1d755c9aa5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/5efa3f6a4e794154a30c785d9a2669e0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/adf5b4194a4f423d8c6d9f5b2ab80c28] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=39.1 K 2024-11-14T02:43:47,770 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:47,770 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b20f5ac91c1463ba1282b1d755c9aa5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731552224425 2024-11-14T02:43:47,771 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5efa3f6a4e794154a30c785d9a2669e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731552224602 2024-11-14T02:43:47,771 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:47,772 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/B is initiating minor compaction (all files) 2024-11-14T02:43:47,772 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/B in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:47,772 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/fba2ef20b40d45ec9d85fd5503a8f32c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/ab7f55473e7b4c43853ff2a9ff3505cd, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/3e5f6655d58142f3bbdedc15a8df3f87] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=36.7 K 2024-11-14T02:43:47,772 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting adf5b4194a4f423d8c6d9f5b2ab80c28, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731552225280 2024-11-14T02:43:47,772 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting fba2ef20b40d45ec9d85fd5503a8f32c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731552224425 2024-11-14T02:43:47,773 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting ab7f55473e7b4c43853ff2a9ff3505cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731552224602 2024-11-14T02:43:47,774 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e5f6655d58142f3bbdedc15a8df3f87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731552225284 2024-11-14T02:43:47,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T02:43:47,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:47,795 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T02:43:47,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:47,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:47,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:47,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:47,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:47,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:47,799 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#A#compaction#65 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:47,799 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/f7727d700a6a4e51bb84e77719f74fdc is 50, key is test_row_0/A:col10/1731552226447/Put/seqid=0 2024-11-14T02:43:47,810 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:47,811 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/c2df8f3bf77a4e2cb7647443d7447fc8 is 50, key is test_row_0/B:col10/1731552226447/Put/seqid=0 2024-11-14T02:43:47,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/4dbcad10f73047c197504c3ee930d49c is 50, key is test_row_0/A:col10/1731552226481/Put/seqid=0 2024-11-14T02:43:47,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741902_1078 (size=13051) 2024-11-14T02:43:47,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741903_1079 (size=13051) 2024-11-14T02:43:47,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741904_1080 (size=12301) 2024-11-14T02:43:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T02:43:48,287 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/f7727d700a6a4e51bb84e77719f74fdc as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/f7727d700a6a4e51bb84e77719f74fdc 2024-11-14T02:43:48,299 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/c2df8f3bf77a4e2cb7647443d7447fc8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c2df8f3bf77a4e2cb7647443d7447fc8 2024-11-14T02:43:48,300 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/A of b0f2fe4f29c62d92b3ee0b2d165c19cb into f7727d700a6a4e51bb84e77719f74fdc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:48,308 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:48,308 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/A, priority=13, startTime=1731552227768; duration=0sec 2024-11-14T02:43:48,308 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:48,309 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:A 2024-11-14T02:43:48,309 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:48,311 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/B of b0f2fe4f29c62d92b3ee0b2d165c19cb into c2df8f3bf77a4e2cb7647443d7447fc8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:48,311 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:48,311 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/B, priority=13, startTime=1731552227770; duration=0sec 2024-11-14T02:43:48,311 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:48,311 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:B 2024-11-14T02:43:48,311 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:48,311 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/C is initiating minor compaction (all files) 2024-11-14T02:43:48,311 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/C in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:48,312 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6daf9839f57a44628c9c4c6b8a6181ff, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/67b2ba534c6842d68c8fef456f275f55, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/af47ed6ebcbf496a9011ac8bd9c746ae] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=36.6 K 2024-11-14T02:43:48,315 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6daf9839f57a44628c9c4c6b8a6181ff, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731552224425 2024-11-14T02:43:48,315 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/4dbcad10f73047c197504c3ee930d49c 2024-11-14T02:43:48,316 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67b2ba534c6842d68c8fef456f275f55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731552224602 2024-11-14T02:43:48,317 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting af47ed6ebcbf496a9011ac8bd9c746ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731552225284 2024-11-14T02:43:48,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/93b1a8c1735d4fc78d938c6d6fa9000c is 50, key is test_row_0/B:col10/1731552226481/Put/seqid=0 2024-11-14T02:43:48,348 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:48,349 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/9675bc296bdf4a1887d6760805f58bd7 is 50, key is test_row_0/C:col10/1731552226447/Put/seqid=0 2024-11-14T02:43:48,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741905_1081 (size=12301) 2024-11-14T02:43:48,374 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/93b1a8c1735d4fc78d938c6d6fa9000c 2024-11-14T02:43:48,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741906_1082 (size=13017) 2024-11-14T02:43:48,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/9994c901640b4e00a8ca19b49bbab4e0 is 50, key is test_row_0/C:col10/1731552226481/Put/seqid=0 2024-11-14T02:43:48,399 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/9675bc296bdf4a1887d6760805f58bd7 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/9675bc296bdf4a1887d6760805f58bd7 2024-11-14T02:43:48,412 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/C of b0f2fe4f29c62d92b3ee0b2d165c19cb into 9675bc296bdf4a1887d6760805f58bd7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:48,412 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:48,412 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/C, priority=13, startTime=1731552227770; duration=0sec 2024-11-14T02:43:48,413 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:48,413 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:C 2024-11-14T02:43:48,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741907_1083 (size=12301) 2024-11-14T02:43:48,421 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/9994c901640b4e00a8ca19b49bbab4e0 2024-11-14T02:43:48,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/4dbcad10f73047c197504c3ee930d49c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4dbcad10f73047c197504c3ee930d49c 2024-11-14T02:43:48,437 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4dbcad10f73047c197504c3ee930d49c, entries=150, sequenceid=520, filesize=12.0 K 2024-11-14T02:43:48,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/93b1a8c1735d4fc78d938c6d6fa9000c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/93b1a8c1735d4fc78d938c6d6fa9000c 2024-11-14T02:43:48,447 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/93b1a8c1735d4fc78d938c6d6fa9000c, entries=150, sequenceid=520, filesize=12.0 K 2024-11-14T02:43:48,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/9994c901640b4e00a8ca19b49bbab4e0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/9994c901640b4e00a8ca19b49bbab4e0 2024-11-14T02:43:48,469 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/9994c901640b4e00a8ca19b49bbab4e0, entries=150, sequenceid=520, filesize=12.0 K 2024-11-14T02:43:48,470 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 675ms, sequenceid=520, compaction requested=false 2024-11-14T02:43:48,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:48,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:48,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-14T02:43:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-14T02:43:48,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-14T02:43:48,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6150 sec 2024-11-14T02:43:48,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6210 sec 2024-11-14T02:43:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:48,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:43:48,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:48,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:48,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:48,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:48,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:48,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:48,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/fdfad36da32e42f480ca7d0c7e75a655 is 50, key is test_row_0/A:col10/1731552228659/Put/seqid=0 2024-11-14T02:43:48,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731552228714, max timestamp=1731552228728Num uniques 859; 2024-11-14T02:43:48,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:43:48,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741908_1084 (size=12301) 2024-11-14T02:43:48,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552288743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:48,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552288745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:48,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:48,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552288749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:48,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552288751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:48,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:48,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:48,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552288856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:48,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:48,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:48,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552288865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:48,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:48,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:48,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552288866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:48,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:48,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:48,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552288866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T02:43:49,006 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:49,006 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-14T02:43:49,009 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T02:43:49,009 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:49,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:49,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552289066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:49,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552289075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:49,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552289076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:49,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:49,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552289076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T02:43:49,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/fdfad36da32e42f480ca7d0c7e75a655 2024-11-14T02:43:49,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/0dfa3d7706764f9ea47c42efe9a85c7d is 50, key is test_row_0/B:col10/1731552228659/Put/seqid=0 2024-11-14T02:43:49,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T02:43:49,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:49,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:49,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:49,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:49,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:49,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741909_1085 (size=12301) 2024-11-14T02:43:49,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T02:43:49,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:49,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:49,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:49,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:49,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:49,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T02:43:49,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:49,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552289375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:49,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:49,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:49,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552289395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:49,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:49,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552289395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:49,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:49,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552289395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:49,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T02:43:49,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:49,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:49,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:49,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:49,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:49,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/0dfa3d7706764f9ea47c42efe9a85c7d 2024-11-14T02:43:49,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T02:43:49,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:49,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:49,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:49,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:49,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T02:43:49,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/740e8a2aca454e889a36820f4e72bfbf is 50, key is test_row_0/C:col10/1731552228659/Put/seqid=0 2024-11-14T02:43:49,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741910_1086 (size=12301) 2024-11-14T02:43:49,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/740e8a2aca454e889a36820f4e72bfbf 2024-11-14T02:43:49,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/fdfad36da32e42f480ca7d0c7e75a655 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/fdfad36da32e42f480ca7d0c7e75a655 2024-11-14T02:43:49,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/fdfad36da32e42f480ca7d0c7e75a655, entries=150, sequenceid=534, filesize=12.0 K 2024-11-14T02:43:49,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/0dfa3d7706764f9ea47c42efe9a85c7d as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/0dfa3d7706764f9ea47c42efe9a85c7d 2024-11-14T02:43:49,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/0dfa3d7706764f9ea47c42efe9a85c7d, entries=150, sequenceid=534, filesize=12.0 K 2024-11-14T02:43:49,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/740e8a2aca454e889a36820f4e72bfbf as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/740e8a2aca454e889a36820f4e72bfbf 2024-11-14T02:43:49,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/740e8a2aca454e889a36820f4e72bfbf, entries=150, sequenceid=534, filesize=12.0 K 2024-11-14T02:43:49,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 1039ms, sequenceid=534, compaction requested=true 2024-11-14T02:43:49,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:49,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:49,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:49,702 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:49,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:49,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:49,702 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:49,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:49,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:49,703 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:49,703 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:49,703 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/B is initiating minor compaction (all files) 2024-11-14T02:43:49,703 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/A is initiating minor compaction (all files) 2024-11-14T02:43:49,703 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/B in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:49,703 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/A in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:49,704 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c2df8f3bf77a4e2cb7647443d7447fc8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/93b1a8c1735d4fc78d938c6d6fa9000c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/0dfa3d7706764f9ea47c42efe9a85c7d] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=36.8 K 2024-11-14T02:43:49,704 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/f7727d700a6a4e51bb84e77719f74fdc, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4dbcad10f73047c197504c3ee930d49c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/fdfad36da32e42f480ca7d0c7e75a655] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=36.8 K 2024-11-14T02:43:49,704 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting c2df8f3bf77a4e2cb7647443d7447fc8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731552225284 2024-11-14T02:43:49,704 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7727d700a6a4e51bb84e77719f74fdc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731552225284 2024-11-14T02:43:49,704 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 93b1a8c1735d4fc78d938c6d6fa9000c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1731552226463 2024-11-14T02:43:49,704 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dbcad10f73047c197504c3ee930d49c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1731552226463 2024-11-14T02:43:49,705 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dfa3d7706764f9ea47c42efe9a85c7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731552228641 2024-11-14T02:43:49,705 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdfad36da32e42f480ca7d0c7e75a655, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731552228641 2024-11-14T02:43:49,716 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#A#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:49,717 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/3cefd8b65e2844a79f9bbad13eaeb702 is 50, key is test_row_0/A:col10/1731552228659/Put/seqid=0 2024-11-14T02:43:49,719 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#B#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:49,719 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/92f3e21c875248bbba538620fb87a9c1 is 50, key is test_row_0/B:col10/1731552228659/Put/seqid=0 2024-11-14T02:43:49,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741911_1087 (size=13153) 2024-11-14T02:43:49,743 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/92f3e21c875248bbba538620fb87a9c1 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/92f3e21c875248bbba538620fb87a9c1 2024-11-14T02:43:49,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741912_1088 (size=13153) 2024-11-14T02:43:49,757 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/B of b0f2fe4f29c62d92b3ee0b2d165c19cb into 92f3e21c875248bbba538620fb87a9c1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:49,757 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:49,757 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/B, priority=13, startTime=1731552229702; duration=0sec 2024-11-14T02:43:49,757 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:49,758 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:B 2024-11-14T02:43:49,758 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:49,760 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:49,760 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/C is initiating minor compaction (all files) 2024-11-14T02:43:49,761 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/C in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:49,761 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/9675bc296bdf4a1887d6760805f58bd7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/9994c901640b4e00a8ca19b49bbab4e0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/740e8a2aca454e889a36820f4e72bfbf] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=36.7 K 2024-11-14T02:43:49,761 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 9675bc296bdf4a1887d6760805f58bd7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731552225284 2024-11-14T02:43:49,762 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 9994c901640b4e00a8ca19b49bbab4e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1731552226463 2024-11-14T02:43:49,762 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 740e8a2aca454e889a36820f4e72bfbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731552228641 2024-11-14T02:43:49,771 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/3cefd8b65e2844a79f9bbad13eaeb702 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/3cefd8b65e2844a79f9bbad13eaeb702 2024-11-14T02:43:49,781 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/A of b0f2fe4f29c62d92b3ee0b2d165c19cb into 3cefd8b65e2844a79f9bbad13eaeb702(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:49,781 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:49,781 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/A, priority=13, startTime=1731552229702; duration=0sec 2024-11-14T02:43:49,781 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:49,781 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:A 2024-11-14T02:43:49,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T02:43:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:49,785 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=125.23 KB heapSize=366.80 KB 2024-11-14T02:43:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:49,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:49,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:49,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:49,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:49,798 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#C#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:49,799 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/03c207c643f14a81b210140e7f2272bf is 50, key is test_row_0/C:col10/1731552228659/Put/seqid=0 2024-11-14T02:43:49,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/1de50e88dba1461fa91132fce18a4589 is 50, key is test_row_0/A:col10/1731552228746/Put/seqid=0 2024-11-14T02:43:49,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741913_1089 (size=13119) 2024-11-14T02:43:49,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741914_1090 (size=12301) 2024-11-14T02:43:49,835 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/1de50e88dba1461fa91132fce18a4589 2024-11-14T02:43:49,848 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/03c207c643f14a81b210140e7f2272bf as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/03c207c643f14a81b210140e7f2272bf 2024-11-14T02:43:49,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/c4c24394a229408fa585ef0294ef04cc is 50, key is test_row_0/B:col10/1731552228746/Put/seqid=0 2024-11-14T02:43:49,862 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/C of b0f2fe4f29c62d92b3ee0b2d165c19cb into 03c207c643f14a81b210140e7f2272bf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:49,862 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:49,862 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/C, priority=13, startTime=1731552229702; duration=0sec 2024-11-14T02:43:49,863 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:49,863 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:C 2024-11-14T02:43:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:49,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:49,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741915_1091 (size=12301) 2024-11-14T02:43:49,906 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/c4c24394a229408fa585ef0294ef04cc 2024-11-14T02:43:49,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/ee523d902a2041749ab8eb897d99526e is 50, key is test_row_0/C:col10/1731552228746/Put/seqid=0 2024-11-14T02:43:49,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552289915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:49,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552289918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:49,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552289922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552289920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:49,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:49,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:49,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741916_1092 (size=12301) 2024-11-14T02:43:49,938 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/ee523d902a2041749ab8eb897d99526e 2024-11-14T02:43:49,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/1de50e88dba1461fa91132fce18a4589 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/1de50e88dba1461fa91132fce18a4589 2024-11-14T02:43:49,961 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/1de50e88dba1461fa91132fce18a4589, entries=150, sequenceid=562, filesize=12.0 K 2024-11-14T02:43:49,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/c4c24394a229408fa585ef0294ef04cc as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c4c24394a229408fa585ef0294ef04cc 2024-11-14T02:43:49,971 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c4c24394a229408fa585ef0294ef04cc, entries=150, sequenceid=562, filesize=12.0 K 2024-11-14T02:43:49,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/ee523d902a2041749ab8eb897d99526e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/ee523d902a2041749ab8eb897d99526e 2024-11-14T02:43:49,981 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/ee523d902a2041749ab8eb897d99526e, entries=150, sequenceid=562, filesize=12.0 K 2024-11-14T02:43:49,982 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~366.75 KB/375552, currentSize=53.67 KB/54960 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 197ms, sequenceid=562, compaction requested=false 2024-11-14T02:43:49,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:49,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:49,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-14T02:43:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-14T02:43:49,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-14T02:43:49,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 974 msec 2024-11-14T02:43:49,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 981 msec 2024-11-14T02:43:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:50,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T02:43:50,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:50,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:50,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:50,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:50,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:50,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:50,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/19c0b9f89dce42faac354a7e30017b28 is 50, key is test_row_0/A:col10/1731552230036/Put/seqid=0 2024-11-14T02:43:50,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741917_1093 (size=9857) 2024-11-14T02:43:50,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552290071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552290071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552290078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552290079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T02:43:50,146 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:50,146 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:43:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-14T02:43:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T02:43:50,149 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:43:50,150 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:43:50,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:43:50,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552290185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552290185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552290195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552290196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T02:43:50,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:50,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:50,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:50,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:50,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:50,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:50,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:50,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552290396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552290396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552290413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552290413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:50,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:50,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:50,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:50,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:50,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T02:43:50,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/19c0b9f89dce42faac354a7e30017b28 2024-11-14T02:43:50,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/4b44ac4706994c1785c19bf905021a34 is 50, key is test_row_0/B:col10/1731552230036/Put/seqid=0 2024-11-14T02:43:50,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741918_1094 (size=9857) 2024-11-14T02:43:50,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:50,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:50,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:50,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:50,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:50,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:50,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552290705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552290716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552290725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552290735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:50,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:50,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:50,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:50,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:50,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:50,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T02:43:50,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:50,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:50,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:50,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:50,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:50,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:50,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/4b44ac4706994c1785c19bf905021a34 2024-11-14T02:43:50,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/59400534e6a44f58abc2647ec7b8498f is 50, key is test_row_0/C:col10/1731552230036/Put/seqid=0 2024-11-14T02:43:51,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741919_1095 (size=9857) 2024-11-14T02:43:51,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/59400534e6a44f58abc2647ec7b8498f 2024-11-14T02:43:51,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/19c0b9f89dce42faac354a7e30017b28 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/19c0b9f89dce42faac354a7e30017b28 2024-11-14T02:43:51,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/19c0b9f89dce42faac354a7e30017b28, entries=100, sequenceid=576, filesize=9.6 K 2024-11-14T02:43:51,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/4b44ac4706994c1785c19bf905021a34 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/4b44ac4706994c1785c19bf905021a34 2024-11-14T02:43:51,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/4b44ac4706994c1785c19bf905021a34, entries=100, sequenceid=576, filesize=9.6 K 2024-11-14T02:43:51,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/59400534e6a44f58abc2647ec7b8498f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/59400534e6a44f58abc2647ec7b8498f 2024-11-14T02:43:51,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:51,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:51,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:51,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:51,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:51,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:51,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/59400534e6a44f58abc2647ec7b8498f, entries=100, sequenceid=576, filesize=9.6 K 2024-11-14T02:43:51,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 1053ms, sequenceid=576, compaction requested=true 2024-11-14T02:43:51,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:51,091 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:51,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:51,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:51,093 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:51,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:51,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:51,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:51,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:51,094 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:51,095 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/A is initiating minor compaction (all files) 2024-11-14T02:43:51,095 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/A in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:51,095 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/3cefd8b65e2844a79f9bbad13eaeb702, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/1de50e88dba1461fa91132fce18a4589, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/19c0b9f89dce42faac354a7e30017b28] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=34.5 K 2024-11-14T02:43:51,096 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cefd8b65e2844a79f9bbad13eaeb702, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731552228641 2024-11-14T02:43:51,096 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1de50e88dba1461fa91132fce18a4589, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731552228734 2024-11-14T02:43:51,097 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19c0b9f89dce42faac354a7e30017b28, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731552229917 2024-11-14T02:43:51,097 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:51,097 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/B is initiating minor compaction (all files) 2024-11-14T02:43:51,097 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/B in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:51,098 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/92f3e21c875248bbba538620fb87a9c1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c4c24394a229408fa585ef0294ef04cc, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/4b44ac4706994c1785c19bf905021a34] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=34.5 K 2024-11-14T02:43:51,099 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f3e21c875248bbba538620fb87a9c1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731552228641 2024-11-14T02:43:51,100 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting c4c24394a229408fa585ef0294ef04cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731552228734 2024-11-14T02:43:51,101 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b44ac4706994c1785c19bf905021a34, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731552229917 2024-11-14T02:43:51,125 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#A#compaction#83 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:51,125 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/80b2c84abc274b64b32c0e9047d86e33 is 50, key is test_row_0/A:col10/1731552230036/Put/seqid=0 2024-11-14T02:43:51,136 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#B#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:51,137 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/2a0dbe7b22c84ddbb8c8dd5a17cf960c is 50, key is test_row_0/B:col10/1731552230036/Put/seqid=0 2024-11-14T02:43:51,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741920_1096 (size=13255) 2024-11-14T02:43:51,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741921_1097 (size=13255) 2024-11-14T02:43:51,198 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/2a0dbe7b22c84ddbb8c8dd5a17cf960c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/2a0dbe7b22c84ddbb8c8dd5a17cf960c 2024-11-14T02:43:51,208 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/B of b0f2fe4f29c62d92b3ee0b2d165c19cb into 2a0dbe7b22c84ddbb8c8dd5a17cf960c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:51,208 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:51,208 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/B, priority=13, startTime=1731552231092; duration=0sec 2024-11-14T02:43:51,208 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:51,208 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:B 2024-11-14T02:43:51,209 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:43:51,211 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:43:51,211 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/C is initiating minor compaction (all files) 2024-11-14T02:43:51,211 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/C in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:51,211 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/03c207c643f14a81b210140e7f2272bf, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/ee523d902a2041749ab8eb897d99526e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/59400534e6a44f58abc2647ec7b8498f] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=34.5 K 2024-11-14T02:43:51,212 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c207c643f14a81b210140e7f2272bf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731552228641 2024-11-14T02:43:51,214 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting ee523d902a2041749ab8eb897d99526e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731552228734 2024-11-14T02:43:51,215 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 59400534e6a44f58abc2647ec7b8498f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731552229917 2024-11-14T02:43:51,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T02:43:51,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:51,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:51,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:51,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:51,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:51,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:51,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:51,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:51,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:51,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:51,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:51,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:51,246 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#C#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:51,247 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/3b57b33515c24269ae23d7bed94f43bd is 50, key is test_row_0/C:col10/1731552230036/Put/seqid=0 2024-11-14T02:43:51,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552291251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/923bb581116a4dd0bd122f668d3a3e5f is 50, key is test_row_0/A:col10/1731552230068/Put/seqid=0 2024-11-14T02:43:51,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552291253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552291256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552291256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T02:43:51,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741922_1098 (size=13221) 2024-11-14T02:43:51,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741923_1099 (size=14741) 2024-11-14T02:43:51,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/923bb581116a4dd0bd122f668d3a3e5f 2024-11-14T02:43:51,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/7c8d371674c2469cbfb08ce0b76b2759 is 50, key is test_row_0/B:col10/1731552230068/Put/seqid=0 2024-11-14T02:43:51,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552291367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552291367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552291376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552291376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741924_1100 (size=12301) 2024-11-14T02:43:51,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/7c8d371674c2469cbfb08ce0b76b2759 2024-11-14T02:43:51,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:51,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:51,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:51,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:51,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:43:51,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/30d1e00f75d14ca7950ac8adb784eef2 is 50, key is test_row_0/C:col10/1731552230068/Put/seqid=0 2024-11-14T02:43:51,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741925_1101 (size=12301) 2024-11-14T02:43:51,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/30d1e00f75d14ca7950ac8adb784eef2 2024-11-14T02:43:51,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/923bb581116a4dd0bd122f668d3a3e5f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/923bb581116a4dd0bd122f668d3a3e5f 2024-11-14T02:43:51,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/923bb581116a4dd0bd122f668d3a3e5f, entries=200, sequenceid=597, filesize=14.4 K 2024-11-14T02:43:51,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/7c8d371674c2469cbfb08ce0b76b2759 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7c8d371674c2469cbfb08ce0b76b2759 2024-11-14T02:43:51,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7c8d371674c2469cbfb08ce0b76b2759, entries=150, sequenceid=597, filesize=12.0 K 2024-11-14T02:43:51,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/30d1e00f75d14ca7950ac8adb784eef2 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/30d1e00f75d14ca7950ac8adb784eef2 2024-11-14T02:43:51,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/30d1e00f75d14ca7950ac8adb784eef2, entries=150, sequenceid=597, filesize=12.0 K 2024-11-14T02:43:51,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 234ms, sequenceid=597, compaction requested=false 2024-11-14T02:43:51,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:51,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T02:43:51,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:51,545 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T02:43:51,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:51,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:51,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:51,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:51,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:51,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:43:51,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/8a1be44746774a3ab2c8f0012d9df4ea is 50, key is test_row_0/A:col10/1731552231241/Put/seqid=0 2024-11-14T02:43:51,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741926_1102 (size=12301) 2024-11-14T02:43:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:51,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. as already flushing 2024-11-14T02:43:51,581 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/80b2c84abc274b64b32c0e9047d86e33 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/80b2c84abc274b64b32c0e9047d86e33 2024-11-14T02:43:51,591 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/A of b0f2fe4f29c62d92b3ee0b2d165c19cb into 80b2c84abc274b64b32c0e9047d86e33(size=12.9 K), total size for store is 27.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:51,591 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:51,591 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/A, priority=13, startTime=1731552231091; duration=0sec 2024-11-14T02:43:51,591 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:51,592 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:A 2024-11-14T02:43:51,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552291654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552291656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552291661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552291667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,718 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/3b57b33515c24269ae23d7bed94f43bd as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3b57b33515c24269ae23d7bed94f43bd 2024-11-14T02:43:51,730 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/C of b0f2fe4f29c62d92b3ee0b2d165c19cb into 3b57b33515c24269ae23d7bed94f43bd(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:51,730 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:51,730 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/C, priority=13, startTime=1731552231093; duration=0sec 2024-11-14T02:43:51,730 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:51,730 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:C 2024-11-14T02:43:51,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552291775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552291778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552291778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552291778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,956 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/8a1be44746774a3ab2c8f0012d9df4ea 2024-11-14T02:43:51,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/4939b51b767a41cdbfeafc15bac553bb is 50, key is test_row_0/B:col10/1731552231241/Put/seqid=0 2024-11-14T02:43:51,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552291986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:51,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552291986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:51,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:51,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:51,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:52,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552291995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:52,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:52,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552291995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:52,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741927_1103 (size=12301) 2024-11-14T02:43:52,024 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/4939b51b767a41cdbfeafc15bac553bb 2024-11-14T02:43:52,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/e4ad07f880e54428a1ad90ec7e46f2f1 is 50, key is test_row_0/C:col10/1731552231241/Put/seqid=0 2024-11-14T02:43:52,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741928_1104 (size=12301) 2024-11-14T02:43:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T02:43:52,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552292306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:52,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552292306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552292306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:52,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731552292307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:52,314 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in b0f2fe4f29c62d92b3ee0b2d165c19cb of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18349 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:43:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552292316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:52,499 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/e4ad07f880e54428a1ad90ec7e46f2f1 2024-11-14T02:43:52,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/8a1be44746774a3ab2c8f0012d9df4ea as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/8a1be44746774a3ab2c8f0012d9df4ea 2024-11-14T02:43:52,518 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/8a1be44746774a3ab2c8f0012d9df4ea, entries=150, sequenceid=609, filesize=12.0 K 2024-11-14T02:43:52,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/4939b51b767a41cdbfeafc15bac553bb as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/4939b51b767a41cdbfeafc15bac553bb 2024-11-14T02:43:52,525 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/4939b51b767a41cdbfeafc15bac553bb, entries=150, sequenceid=609, filesize=12.0 K 2024-11-14T02:43:52,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/e4ad07f880e54428a1ad90ec7e46f2f1 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/e4ad07f880e54428a1ad90ec7e46f2f1 2024-11-14T02:43:52,543 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/e4ad07f880e54428a1ad90ec7e46f2f1, entries=150, sequenceid=609, filesize=12.0 K 2024-11-14T02:43:52,547 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 1002ms, sequenceid=609, compaction requested=true 2024-11-14T02:43:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-14T02:43:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-14T02:43:52,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-14T02:43:52,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3990 sec 2024-11-14T02:43:52,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.4090 sec 2024-11-14T02:43:52,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552231651, max timestamp=1731552232823Num uniques 859; 2024-11-14T02:43:52,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:43:52,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552231651, max timestamp=1731552232823Num uniques 859; 2024-11-14T02:43:52,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:43:52,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552231651, max timestamp=1731552232823Num uniques 859; 2024-11-14T02:43:52,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:43:52,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-14T02:43:52,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:52,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:52,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:52,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:52,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:52,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:52,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/bc0f58b024694ded931f21124ff18755 is 50, key is test_row_0/A:col10/1731552232842/Put/seqid=0 2024-11-14T02:43:52,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741929_1105 (size=17181) 2024-11-14T02:43:52,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731552232873, max timestamp=1731552232898Num uniques 859; 2024-11-14T02:43:52,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:43:52,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552232873, max timestamp=1731552232900Num uniques 859; 2024-11-14T02:43:52,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:43:52,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552232873, max timestamp=1731552232900Num uniques 859; 2024-11-14T02:43:52,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:43:52,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552292928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:52,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552292931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552292933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:52,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:52,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552292944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:52,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:52,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:53,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552293056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:53,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:53,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552293056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:53,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:53,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552293056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:53,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552293056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:53,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:53,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552293275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:53,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:53,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552293275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:53,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:53,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552293276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:53,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552293276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:53,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=637 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/bc0f58b024694ded931f21124ff18755 2024-11-14T02:43:53,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/556cb7697bcb4b5e896df142203290ad is 50, key is test_row_0/B:col10/1731552232842/Put/seqid=0 2024-11-14T02:43:53,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741930_1106 (size=12301) 2024-11-14T02:43:53,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:53,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55004 deadline: 1731552293585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:53,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:53,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1731552293585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:53,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731552293585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:53,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:43:53,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731552293595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:43:53,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:43:53,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:43:53,609 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-14T02:43:53,609 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-14T02:43:53,609 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:53,609 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:53,613 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-14T02:43:53,614 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:53,614 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:53,614 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:53,614 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:53,614 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:53,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:53,614 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:53,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:53,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:53,615 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:53,623 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-14T02:43:53,623 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:53,623 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:53,623 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:53,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:53,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=637 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/556cb7697bcb4b5e896df142203290ad 2024-11-14T02:43:53,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/6ff0027c1af74a94a57023087b71d0c6 is 50, key is test_row_0/C:col10/1731552232842/Put/seqid=0 2024-11-14T02:43:53,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741931_1107 (size=12301) 2024-11-14T02:43:53,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=637 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/6ff0027c1af74a94a57023087b71d0c6 2024-11-14T02:43:53,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/bc0f58b024694ded931f21124ff18755 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/bc0f58b024694ded931f21124ff18755 2024-11-14T02:43:53,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/bc0f58b024694ded931f21124ff18755, entries=250, sequenceid=637, filesize=16.8 K 2024-11-14T02:43:53,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/556cb7697bcb4b5e896df142203290ad as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/556cb7697bcb4b5e896df142203290ad 2024-11-14T02:43:53,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/556cb7697bcb4b5e896df142203290ad, entries=150, sequenceid=637, filesize=12.0 K 2024-11-14T02:43:53,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/6ff0027c1af74a94a57023087b71d0c6 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6ff0027c1af74a94a57023087b71d0c6 2024-11-14T02:43:53,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6ff0027c1af74a94a57023087b71d0c6, entries=150, sequenceid=637, filesize=12.0 K 2024-11-14T02:43:53,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=125.23 KB/128240 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 945ms, sequenceid=637, compaction requested=true 2024-11-14T02:43:53,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:53,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:43:53,788 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:53,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:53,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:43:53,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:53,788 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:53,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b0f2fe4f29c62d92b3ee0b2d165c19cb:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:43:53,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:53,790 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:53,790 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57478 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:53,790 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/B is initiating minor compaction (all files) 2024-11-14T02:43:53,790 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/A is initiating minor compaction (all files) 2024-11-14T02:43:53,790 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/A in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:53,790 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/B in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:53,790 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/2a0dbe7b22c84ddbb8c8dd5a17cf960c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7c8d371674c2469cbfb08ce0b76b2759, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/4939b51b767a41cdbfeafc15bac553bb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/556cb7697bcb4b5e896df142203290ad] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=49.0 K 2024-11-14T02:43:53,790 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/80b2c84abc274b64b32c0e9047d86e33, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/923bb581116a4dd0bd122f668d3a3e5f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/8a1be44746774a3ab2c8f0012d9df4ea, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/bc0f58b024694ded931f21124ff18755] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=56.1 K 2024-11-14T02:43:53,791 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b2c84abc274b64b32c0e9047d86e33, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731552228749 2024-11-14T02:43:53,791 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a0dbe7b22c84ddbb8c8dd5a17cf960c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731552228749 2024-11-14T02:43:53,791 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 923bb581116a4dd0bd122f668d3a3e5f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1731552230067 2024-11-14T02:43:53,791 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c8d371674c2469cbfb08ce0b76b2759, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1731552230067 2024-11-14T02:43:53,792 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a1be44746774a3ab2c8f0012d9df4ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1731552231230 2024-11-14T02:43:53,792 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 4939b51b767a41cdbfeafc15bac553bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1731552231230 2024-11-14T02:43:53,792 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 556cb7697bcb4b5e896df142203290ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=637, earliestPutTs=1731552231656 2024-11-14T02:43:53,792 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc0f58b024694ded931f21124ff18755, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=637, earliestPutTs=1731552231656 2024-11-14T02:43:53,803 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#B#compaction#95 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:53,803 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/2e90fff25050440a88306ac3b03cfd90 is 50, key is test_row_0/B:col10/1731552232842/Put/seqid=0 2024-11-14T02:43:53,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741932_1108 (size=13391) 2024-11-14T02:43:53,820 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/2e90fff25050440a88306ac3b03cfd90 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/2e90fff25050440a88306ac3b03cfd90 2024-11-14T02:43:53,827 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/B of b0f2fe4f29c62d92b3ee0b2d165c19cb into 2e90fff25050440a88306ac3b03cfd90(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:53,827 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:53,827 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/B, priority=12, startTime=1731552233788; duration=0sec 2024-11-14T02:43:53,827 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:43:53,827 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:B 2024-11-14T02:43:53,827 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:43:53,829 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:43:53,829 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): b0f2fe4f29c62d92b3ee0b2d165c19cb/C is initiating minor compaction (all files) 2024-11-14T02:43:53,829 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b0f2fe4f29c62d92b3ee0b2d165c19cb/C in TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:43:53,829 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3b57b33515c24269ae23d7bed94f43bd, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/30d1e00f75d14ca7950ac8adb784eef2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/e4ad07f880e54428a1ad90ec7e46f2f1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6ff0027c1af74a94a57023087b71d0c6] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp, totalSize=48.9 K 2024-11-14T02:43:53,830 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b57b33515c24269ae23d7bed94f43bd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731552228749 2024-11-14T02:43:53,830 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 30d1e00f75d14ca7950ac8adb784eef2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1731552230067 2024-11-14T02:43:53,831 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting e4ad07f880e54428a1ad90ec7e46f2f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1731552231230 2024-11-14T02:43:53,831 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#A#compaction#96 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:53,831 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ff0027c1af74a94a57023087b71d0c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=637, earliestPutTs=1731552231656 2024-11-14T02:43:53,831 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/fcac97f732584d13b9ff7a9e817f5664 is 50, key is test_row_0/A:col10/1731552232842/Put/seqid=0 2024-11-14T02:43:53,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741933_1109 (size=13391) 2024-11-14T02:43:53,844 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/fcac97f732584d13b9ff7a9e817f5664 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/fcac97f732584d13b9ff7a9e817f5664 2024-11-14T02:43:53,851 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/A of b0f2fe4f29c62d92b3ee0b2d165c19cb into fcac97f732584d13b9ff7a9e817f5664(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:53,851 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:53,851 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/A, priority=12, startTime=1731552233788; duration=0sec 2024-11-14T02:43:53,851 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:53,851 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:A 2024-11-14T02:43:53,853 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b0f2fe4f29c62d92b3ee0b2d165c19cb#C#compaction#97 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:43:53,853 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/67de34a9312e4f91979ad5343cc02ed0 is 50, key is test_row_0/C:col10/1731552232842/Put/seqid=0 2024-11-14T02:43:53,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741934_1110 (size=13357) 2024-11-14T02:43:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:43:54,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=145.36 KB heapSize=415.77 KB 2024-11-14T02:43:54,104 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-14T02:43:54,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731552232941, max timestamp=1731552234096Num uniques 1018; 2024-11-14T02:43:54,104 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:54,104 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:54,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-14T02:43:54,105 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:54,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:43:54,106 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-14T02:43:54,106 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T02:43:54,106 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:43:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:43:54,106 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:43:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T02:43:54,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:54,107 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-14T02:43:54,107 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:54,107 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:54,107 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:54,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:54,109 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-14T02:43:54,109 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:43:54,109 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:54,109 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:43:54,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:43:54,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/3332d9bad0e94e31ba80b9599cb9fdb8 is 50, key is test_row_0/A:col10/1731552234096/Put/seqid=0 2024-11-14T02:43:54,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741935_1111 (size=12301) 2024-11-14T02:43:54,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=677 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/3332d9bad0e94e31ba80b9599cb9fdb8 2024-11-14T02:43:54,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/1aa3ae73f30c4fcb8b905a828b8958c2 is 50, key is test_row_0/B:col10/1731552234096/Put/seqid=0 2024-11-14T02:43:54,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741936_1112 (size=12301) 2024-11-14T02:43:54,286 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/67de34a9312e4f91979ad5343cc02ed0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/67de34a9312e4f91979ad5343cc02ed0 2024-11-14T02:43:54,314 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b0f2fe4f29c62d92b3ee0b2d165c19cb/C of b0f2fe4f29c62d92b3ee0b2d165c19cb into 67de34a9312e4f91979ad5343cc02ed0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:43:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T02:43:54,316 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:43:54,317 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:54,317 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., storeName=b0f2fe4f29c62d92b3ee0b2d165c19cb/C, priority=12, startTime=1731552233788; duration=0sec 2024-11-14T02:43:54,321 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:43:54,321 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b0f2fe4f29c62d92b3ee0b2d165c19cb:C 2024-11-14T02:43:54,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.15 KB at sequenceid=677 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/1aa3ae73f30c4fcb8b905a828b8958c2 2024-11-14T02:43:54,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/60f7531360ba48f5a36119a46ca31297 is 50, key is test_row_0/C:col10/1731552234096/Put/seqid=0 2024-11-14T02:43:54,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741937_1113 (size=12301) 2024-11-14T02:43:54,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=677 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/60f7531360ba48f5a36119a46ca31297 2024-11-14T02:43:54,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/3332d9bad0e94e31ba80b9599cb9fdb8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/3332d9bad0e94e31ba80b9599cb9fdb8 2024-11-14T02:43:54,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/3332d9bad0e94e31ba80b9599cb9fdb8, entries=150, sequenceid=677, filesize=12.0 K 2024-11-14T02:43:54,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/1aa3ae73f30c4fcb8b905a828b8958c2 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/1aa3ae73f30c4fcb8b905a828b8958c2 2024-11-14T02:43:54,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/1aa3ae73f30c4fcb8b905a828b8958c2, entries=150, sequenceid=677, filesize=12.0 K 2024-11-14T02:43:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/60f7531360ba48f5a36119a46ca31297 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/60f7531360ba48f5a36119a46ca31297 2024-11-14T02:43:54,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/60f7531360ba48f5a36119a46ca31297, entries=150, sequenceid=677, filesize=12.0 K 2024-11-14T02:43:54,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~106.54 KB/109102, heapSize ~300.48 KB/307696, currentSize=6.71 KB/6870 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 582ms, sequenceid=677, compaction requested=false 2024-11-14T02:43:54,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: 2024-11-14T02:43:55,195 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T02:44:02,388 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-14T02:44:02,388 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:02,388 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:02,388 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:02,388 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:02,388 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T02:44:02,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-14T02:44:02,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 135 2024-11-14T02:44:02,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 137 2024-11-14T02:44:02,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 144 2024-11-14T02:44:02,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 151 2024-11-14T02:44:02,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T02:44:02,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2099 2024-11-14T02:44:02,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2107 2024-11-14T02:44:02,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T02:44:02,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 971 2024-11-14T02:44:02,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2912 rows 2024-11-14T02:44:02,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 971 2024-11-14T02:44:02,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2912 rows 2024-11-14T02:44:02,389 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:44:02,390 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:44:02,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:02,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:02,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:02,396 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T02:44:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T02:44:02,405 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552242405"}]},"ts":"1731552242405"} 2024-11-14T02:44:02,408 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T02:44:02,408 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T02:44:02,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:44:02,415 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b0f2fe4f29c62d92b3ee0b2d165c19cb, UNASSIGN}] 2024-11-14T02:44:02,416 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b0f2fe4f29c62d92b3ee0b2d165c19cb, UNASSIGN 2024-11-14T02:44:02,417 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b0f2fe4f29c62d92b3ee0b2d165c19cb, regionState=CLOSING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:02,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b0f2fe4f29c62d92b3ee0b2d165c19cb, UNASSIGN because future has completed 2024-11-14T02:44:02,420 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:44:02,420 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:44:02,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T02:44:02,577 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:44:02,578 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:44:02,578 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing b0f2fe4f29c62d92b3ee0b2d165c19cb, disabling compactions & flushes 2024-11-14T02:44:02,579 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:44:02,579 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:44:02,579 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. after waiting 0 ms 2024-11-14T02:44:02,579 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:44:02,579 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-14T02:44:02,579 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=A 2024-11-14T02:44:02,579 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:02,579 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=B 2024-11-14T02:44:02,579 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:02,579 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b0f2fe4f29c62d92b3ee0b2d165c19cb, store=C 2024-11-14T02:44:02,580 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:02,585 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/41959445d32a426cb7132f89bb531564 is 50, key is test_row_1/A:col10/1731552242385/Put/seqid=0 2024-11-14T02:44:02,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741938_1114 (size=9857) 2024-11-14T02:44:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T02:44:02,990 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=683 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/41959445d32a426cb7132f89bb531564 2024-11-14T02:44:02,998 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/6fefba764f3242be966bc437aa76d7d2 is 50, key is test_row_1/B:col10/1731552242385/Put/seqid=0 2024-11-14T02:44:03,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741939_1115 (size=9857) 2024-11-14T02:44:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T02:44:03,403 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=683 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/6fefba764f3242be966bc437aa76d7d2 2024-11-14T02:44:03,414 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/d7ded38e6c2f43be82c2d8dd7473a79f is 50, key is test_row_1/C:col10/1731552242385/Put/seqid=0 2024-11-14T02:44:03,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741940_1116 (size=9857) 2024-11-14T02:44:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T02:44:03,819 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=683 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/d7ded38e6c2f43be82c2d8dd7473a79f 2024-11-14T02:44:03,824 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/A/41959445d32a426cb7132f89bb531564 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/41959445d32a426cb7132f89bb531564 2024-11-14T02:44:03,829 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/41959445d32a426cb7132f89bb531564, entries=100, sequenceid=683, filesize=9.6 K 2024-11-14T02:44:03,830 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/B/6fefba764f3242be966bc437aa76d7d2 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/6fefba764f3242be966bc437aa76d7d2 2024-11-14T02:44:03,834 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/6fefba764f3242be966bc437aa76d7d2, entries=100, sequenceid=683, filesize=9.6 K 2024-11-14T02:44:03,835 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/.tmp/C/d7ded38e6c2f43be82c2d8dd7473a79f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/d7ded38e6c2f43be82c2d8dd7473a79f 2024-11-14T02:44:03,841 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/d7ded38e6c2f43be82c2d8dd7473a79f, entries=100, sequenceid=683, filesize=9.6 K 2024-11-14T02:44:03,842 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 1263ms, sequenceid=683, compaction requested=true 2024-11-14T02:44:03,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/449dde5541904740a3407373eb48e0dc, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/fd710b97265c4c44b8c621e3141b56a6, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/13e764296a814371b6c8239a7b04e222, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/2c0a765d035047828064ab46a8987035, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/125486a6a79244f6b809edd8386f4466, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/2cd289e7e9834a15a94fd402d3ee0703, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4fa7ed0a3da64386aaf019be8fa167c5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a0639cdd245a4f65a6393a775e23db96, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/c2db6a08cf7a4a43ad7885b71704e73b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a753feb7abc84ade8592763308b13577, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/57b88ad03d904fe08ff2f485473653b5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a36fa4a8a6bd40bab6a2bb889493f3f7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a74556c26f5d469b812bad2895e81395, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4b6fab1f9bf44db996e308e08c21637d, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/5413a6c151074dbc9af41d43202841a3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/8f2fb4b3d79f45c0a215f0612e93b8c9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/bc8821673a674cc59e93b32542dc5c84, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/11fa5577bb3944e584ae55b4d8b2311b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/9e0e1e0da97d48e085a782141cc93484, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/7b20f5ac91c1463ba1282b1d755c9aa5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/5efa3f6a4e794154a30c785d9a2669e0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/adf5b4194a4f423d8c6d9f5b2ab80c28, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/f7727d700a6a4e51bb84e77719f74fdc, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4dbcad10f73047c197504c3ee930d49c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/3cefd8b65e2844a79f9bbad13eaeb702, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/fdfad36da32e42f480ca7d0c7e75a655, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/1de50e88dba1461fa91132fce18a4589, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/80b2c84abc274b64b32c0e9047d86e33, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/19c0b9f89dce42faac354a7e30017b28, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/923bb581116a4dd0bd122f668d3a3e5f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/8a1be44746774a3ab2c8f0012d9df4ea, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/bc0f58b024694ded931f21124ff18755] to archive 2024-11-14T02:44:03,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:44:03,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/449dde5541904740a3407373eb48e0dc to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/449dde5541904740a3407373eb48e0dc 2024-11-14T02:44:03,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/fd710b97265c4c44b8c621e3141b56a6 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/fd710b97265c4c44b8c621e3141b56a6 2024-11-14T02:44:03,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/13e764296a814371b6c8239a7b04e222 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/13e764296a814371b6c8239a7b04e222 2024-11-14T02:44:03,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/2c0a765d035047828064ab46a8987035 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/2c0a765d035047828064ab46a8987035 2024-11-14T02:44:03,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/125486a6a79244f6b809edd8386f4466 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/125486a6a79244f6b809edd8386f4466 2024-11-14T02:44:03,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/2cd289e7e9834a15a94fd402d3ee0703 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/2cd289e7e9834a15a94fd402d3ee0703 2024-11-14T02:44:03,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4fa7ed0a3da64386aaf019be8fa167c5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4fa7ed0a3da64386aaf019be8fa167c5 2024-11-14T02:44:03,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a0639cdd245a4f65a6393a775e23db96 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a0639cdd245a4f65a6393a775e23db96 2024-11-14T02:44:03,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/c2db6a08cf7a4a43ad7885b71704e73b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/c2db6a08cf7a4a43ad7885b71704e73b 2024-11-14T02:44:03,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a753feb7abc84ade8592763308b13577 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a753feb7abc84ade8592763308b13577 2024-11-14T02:44:03,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/57b88ad03d904fe08ff2f485473653b5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/57b88ad03d904fe08ff2f485473653b5 2024-11-14T02:44:03,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a36fa4a8a6bd40bab6a2bb889493f3f7 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a36fa4a8a6bd40bab6a2bb889493f3f7 2024-11-14T02:44:03,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a74556c26f5d469b812bad2895e81395 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/a74556c26f5d469b812bad2895e81395 2024-11-14T02:44:03,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4b6fab1f9bf44db996e308e08c21637d to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4b6fab1f9bf44db996e308e08c21637d 2024-11-14T02:44:03,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/5413a6c151074dbc9af41d43202841a3 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/5413a6c151074dbc9af41d43202841a3 2024-11-14T02:44:03,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/8f2fb4b3d79f45c0a215f0612e93b8c9 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/8f2fb4b3d79f45c0a215f0612e93b8c9 2024-11-14T02:44:03,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/bc8821673a674cc59e93b32542dc5c84 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/bc8821673a674cc59e93b32542dc5c84 2024-11-14T02:44:03,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/11fa5577bb3944e584ae55b4d8b2311b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/11fa5577bb3944e584ae55b4d8b2311b 2024-11-14T02:44:03,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/9e0e1e0da97d48e085a782141cc93484 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/9e0e1e0da97d48e085a782141cc93484 2024-11-14T02:44:03,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/7b20f5ac91c1463ba1282b1d755c9aa5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/7b20f5ac91c1463ba1282b1d755c9aa5 2024-11-14T02:44:03,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/5efa3f6a4e794154a30c785d9a2669e0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/5efa3f6a4e794154a30c785d9a2669e0 2024-11-14T02:44:03,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/adf5b4194a4f423d8c6d9f5b2ab80c28 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/adf5b4194a4f423d8c6d9f5b2ab80c28 2024-11-14T02:44:03,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/f7727d700a6a4e51bb84e77719f74fdc to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/f7727d700a6a4e51bb84e77719f74fdc 2024-11-14T02:44:03,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4dbcad10f73047c197504c3ee930d49c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/4dbcad10f73047c197504c3ee930d49c 2024-11-14T02:44:03,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/3cefd8b65e2844a79f9bbad13eaeb702 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/3cefd8b65e2844a79f9bbad13eaeb702 2024-11-14T02:44:03,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/fdfad36da32e42f480ca7d0c7e75a655 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/fdfad36da32e42f480ca7d0c7e75a655 2024-11-14T02:44:03,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/1de50e88dba1461fa91132fce18a4589 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/1de50e88dba1461fa91132fce18a4589 2024-11-14T02:44:03,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/80b2c84abc274b64b32c0e9047d86e33 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/80b2c84abc274b64b32c0e9047d86e33 2024-11-14T02:44:03,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/19c0b9f89dce42faac354a7e30017b28 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/19c0b9f89dce42faac354a7e30017b28 2024-11-14T02:44:03,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/923bb581116a4dd0bd122f668d3a3e5f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/923bb581116a4dd0bd122f668d3a3e5f 2024-11-14T02:44:03,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/8a1be44746774a3ab2c8f0012d9df4ea to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/8a1be44746774a3ab2c8f0012d9df4ea 2024-11-14T02:44:03,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/bc0f58b024694ded931f21124ff18755 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/bc0f58b024694ded931f21124ff18755 2024-11-14T02:44:03,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/534a6ac05d5f464aac5382d35fb169c9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/39f11951602245d8acea2a4e9de1dbe0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d4256978f69746b4bb83ed3c746d97f3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/9c5c5b87768f4803afa791b63bb835b5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/977c3b9088be48a4a4f1b83e6cf15282, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7028ef20e4374831a2a6bc98569212da, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/ae0f169c8d604300aec5dcaf18d3b63d, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/871ed68db2924844b7c968888219fc16, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/efdbf7535667489cbe640f7bb533f519, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/5e9735171df045f3a9bcdda9a1dfb1ac, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/3ce1067af9c44f158cda85d70afdf288, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/07c5ef1eb8cc4a359a13f1db5aa688b3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c76cd0eec6654d3f8127c4fe78133788, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7053a56c6bc54d74847452b7b4a5c1cc, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c7d33a94c5984d3a8194d7989cf5adce, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d71350b894cf4815b1bcf7a317ab8507, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/52d34335ac5e4e5db56b03b6f89938e0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d1e7b70eb0f34b40a9e0b453bbb1ae3e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/fba2ef20b40d45ec9d85fd5503a8f32c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/fd05f3eefdb8414daf21b329bc82c2fc, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/ab7f55473e7b4c43853ff2a9ff3505cd, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c2df8f3bf77a4e2cb7647443d7447fc8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/3e5f6655d58142f3bbdedc15a8df3f87, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/93b1a8c1735d4fc78d938c6d6fa9000c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/92f3e21c875248bbba538620fb87a9c1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/0dfa3d7706764f9ea47c42efe9a85c7d, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c4c24394a229408fa585ef0294ef04cc, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/2a0dbe7b22c84ddbb8c8dd5a17cf960c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/4b44ac4706994c1785c19bf905021a34, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7c8d371674c2469cbfb08ce0b76b2759, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/4939b51b767a41cdbfeafc15bac553bb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/556cb7697bcb4b5e896df142203290ad] to archive 2024-11-14T02:44:03,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:44:03,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/534a6ac05d5f464aac5382d35fb169c9 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/534a6ac05d5f464aac5382d35fb169c9 2024-11-14T02:44:03,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/39f11951602245d8acea2a4e9de1dbe0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/39f11951602245d8acea2a4e9de1dbe0 2024-11-14T02:44:03,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d4256978f69746b4bb83ed3c746d97f3 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d4256978f69746b4bb83ed3c746d97f3 2024-11-14T02:44:03,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/9c5c5b87768f4803afa791b63bb835b5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/9c5c5b87768f4803afa791b63bb835b5 2024-11-14T02:44:03,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/977c3b9088be48a4a4f1b83e6cf15282 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/977c3b9088be48a4a4f1b83e6cf15282 2024-11-14T02:44:03,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7028ef20e4374831a2a6bc98569212da to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7028ef20e4374831a2a6bc98569212da 2024-11-14T02:44:03,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/ae0f169c8d604300aec5dcaf18d3b63d to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/ae0f169c8d604300aec5dcaf18d3b63d 2024-11-14T02:44:03,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/871ed68db2924844b7c968888219fc16 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/871ed68db2924844b7c968888219fc16 2024-11-14T02:44:03,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/efdbf7535667489cbe640f7bb533f519 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/efdbf7535667489cbe640f7bb533f519 2024-11-14T02:44:03,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/5e9735171df045f3a9bcdda9a1dfb1ac to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/5e9735171df045f3a9bcdda9a1dfb1ac 2024-11-14T02:44:03,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/3ce1067af9c44f158cda85d70afdf288 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/3ce1067af9c44f158cda85d70afdf288 2024-11-14T02:44:03,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/07c5ef1eb8cc4a359a13f1db5aa688b3 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/07c5ef1eb8cc4a359a13f1db5aa688b3 2024-11-14T02:44:03,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c76cd0eec6654d3f8127c4fe78133788 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c76cd0eec6654d3f8127c4fe78133788 2024-11-14T02:44:03,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7053a56c6bc54d74847452b7b4a5c1cc to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7053a56c6bc54d74847452b7b4a5c1cc 2024-11-14T02:44:03,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c7d33a94c5984d3a8194d7989cf5adce to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c7d33a94c5984d3a8194d7989cf5adce 2024-11-14T02:44:03,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d71350b894cf4815b1bcf7a317ab8507 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d71350b894cf4815b1bcf7a317ab8507 2024-11-14T02:44:03,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/52d34335ac5e4e5db56b03b6f89938e0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/52d34335ac5e4e5db56b03b6f89938e0 2024-11-14T02:44:03,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d1e7b70eb0f34b40a9e0b453bbb1ae3e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/d1e7b70eb0f34b40a9e0b453bbb1ae3e 2024-11-14T02:44:03,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/fba2ef20b40d45ec9d85fd5503a8f32c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/fba2ef20b40d45ec9d85fd5503a8f32c 2024-11-14T02:44:03,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/fd05f3eefdb8414daf21b329bc82c2fc to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/fd05f3eefdb8414daf21b329bc82c2fc 2024-11-14T02:44:03,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/ab7f55473e7b4c43853ff2a9ff3505cd to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/ab7f55473e7b4c43853ff2a9ff3505cd 2024-11-14T02:44:03,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c2df8f3bf77a4e2cb7647443d7447fc8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c2df8f3bf77a4e2cb7647443d7447fc8 2024-11-14T02:44:03,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/3e5f6655d58142f3bbdedc15a8df3f87 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/3e5f6655d58142f3bbdedc15a8df3f87 2024-11-14T02:44:03,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/93b1a8c1735d4fc78d938c6d6fa9000c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/93b1a8c1735d4fc78d938c6d6fa9000c 2024-11-14T02:44:03,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/92f3e21c875248bbba538620fb87a9c1 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/92f3e21c875248bbba538620fb87a9c1 2024-11-14T02:44:03,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/0dfa3d7706764f9ea47c42efe9a85c7d to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/0dfa3d7706764f9ea47c42efe9a85c7d 2024-11-14T02:44:03,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c4c24394a229408fa585ef0294ef04cc to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/c4c24394a229408fa585ef0294ef04cc 2024-11-14T02:44:03,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/2a0dbe7b22c84ddbb8c8dd5a17cf960c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/2a0dbe7b22c84ddbb8c8dd5a17cf960c 2024-11-14T02:44:03,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/4b44ac4706994c1785c19bf905021a34 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/4b44ac4706994c1785c19bf905021a34 2024-11-14T02:44:03,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7c8d371674c2469cbfb08ce0b76b2759 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/7c8d371674c2469cbfb08ce0b76b2759 2024-11-14T02:44:03,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/4939b51b767a41cdbfeafc15bac553bb to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/4939b51b767a41cdbfeafc15bac553bb 2024-11-14T02:44:03,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/556cb7697bcb4b5e896df142203290ad to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/556cb7697bcb4b5e896df142203290ad 2024-11-14T02:44:03,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/d479c901466343e4855dc9353fa9377e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/f974d45410ec41d9b68452a26761f179, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/4854ed0a285041a2b04eb28e6972c48f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/7c79168ab0a44be1bbca12544587daf8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/334acba7ae8447be9eff06493eb535ae, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3afcd246c260451a90f7b08e98309f71, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6770abe80c184b6bb5ee525d17e38e9b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/2c98dc7864554a689ec2de8b7e2ffd24, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/194c39adf43e4ba4969050e3d78fd9d5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/596226adb2f44920be3826732900c0d1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/0fecab8a02284e948263ce8e73067633, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/10bc312a8fe9457f995d4673729596f5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/0fe4eb51683048e1a82ea848080e9cf2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/c2d0eeab8d544d3aa718054f91f3af45, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/b7271eb02cf94c32a5dd57026abf6bc8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3b7ce1c1931c4651b01325f5f3dc7e4b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/cdc969c4ea8841cea16db4df75d90708, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6daf9839f57a44628c9c4c6b8a6181ff, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/114f30f356414f318a3bcc0cb7b34b78, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/67b2ba534c6842d68c8fef456f275f55, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/9675bc296bdf4a1887d6760805f58bd7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/af47ed6ebcbf496a9011ac8bd9c746ae, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/9994c901640b4e00a8ca19b49bbab4e0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/03c207c643f14a81b210140e7f2272bf, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/740e8a2aca454e889a36820f4e72bfbf, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/ee523d902a2041749ab8eb897d99526e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3b57b33515c24269ae23d7bed94f43bd, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/59400534e6a44f58abc2647ec7b8498f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/30d1e00f75d14ca7950ac8adb784eef2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/e4ad07f880e54428a1ad90ec7e46f2f1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6ff0027c1af74a94a57023087b71d0c6] to archive 2024-11-14T02:44:03,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:44:03,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/d479c901466343e4855dc9353fa9377e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/d479c901466343e4855dc9353fa9377e 2024-11-14T02:44:03,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/f974d45410ec41d9b68452a26761f179 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/f974d45410ec41d9b68452a26761f179 2024-11-14T02:44:03,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/4854ed0a285041a2b04eb28e6972c48f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/4854ed0a285041a2b04eb28e6972c48f 2024-11-14T02:44:03,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/7c79168ab0a44be1bbca12544587daf8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/7c79168ab0a44be1bbca12544587daf8 2024-11-14T02:44:03,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/334acba7ae8447be9eff06493eb535ae to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/334acba7ae8447be9eff06493eb535ae 2024-11-14T02:44:03,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3afcd246c260451a90f7b08e98309f71 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3afcd246c260451a90f7b08e98309f71 2024-11-14T02:44:03,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6770abe80c184b6bb5ee525d17e38e9b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6770abe80c184b6bb5ee525d17e38e9b 2024-11-14T02:44:03,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/2c98dc7864554a689ec2de8b7e2ffd24 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/2c98dc7864554a689ec2de8b7e2ffd24 2024-11-14T02:44:03,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/194c39adf43e4ba4969050e3d78fd9d5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/194c39adf43e4ba4969050e3d78fd9d5 2024-11-14T02:44:03,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/596226adb2f44920be3826732900c0d1 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/596226adb2f44920be3826732900c0d1 2024-11-14T02:44:03,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/0fecab8a02284e948263ce8e73067633 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/0fecab8a02284e948263ce8e73067633 2024-11-14T02:44:03,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/10bc312a8fe9457f995d4673729596f5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/10bc312a8fe9457f995d4673729596f5 2024-11-14T02:44:03,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/0fe4eb51683048e1a82ea848080e9cf2 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/0fe4eb51683048e1a82ea848080e9cf2 2024-11-14T02:44:03,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/c2d0eeab8d544d3aa718054f91f3af45 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/c2d0eeab8d544d3aa718054f91f3af45 2024-11-14T02:44:03,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/b7271eb02cf94c32a5dd57026abf6bc8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/b7271eb02cf94c32a5dd57026abf6bc8 2024-11-14T02:44:03,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3b7ce1c1931c4651b01325f5f3dc7e4b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3b7ce1c1931c4651b01325f5f3dc7e4b 2024-11-14T02:44:03,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/cdc969c4ea8841cea16db4df75d90708 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/cdc969c4ea8841cea16db4df75d90708 2024-11-14T02:44:03,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6daf9839f57a44628c9c4c6b8a6181ff to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6daf9839f57a44628c9c4c6b8a6181ff 2024-11-14T02:44:03,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/114f30f356414f318a3bcc0cb7b34b78 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/114f30f356414f318a3bcc0cb7b34b78 2024-11-14T02:44:03,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/67b2ba534c6842d68c8fef456f275f55 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/67b2ba534c6842d68c8fef456f275f55 2024-11-14T02:44:03,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/9675bc296bdf4a1887d6760805f58bd7 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/9675bc296bdf4a1887d6760805f58bd7 2024-11-14T02:44:03,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/af47ed6ebcbf496a9011ac8bd9c746ae to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/af47ed6ebcbf496a9011ac8bd9c746ae 2024-11-14T02:44:03,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/9994c901640b4e00a8ca19b49bbab4e0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/9994c901640b4e00a8ca19b49bbab4e0 2024-11-14T02:44:03,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/03c207c643f14a81b210140e7f2272bf to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/03c207c643f14a81b210140e7f2272bf 2024-11-14T02:44:03,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/740e8a2aca454e889a36820f4e72bfbf to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/740e8a2aca454e889a36820f4e72bfbf 2024-11-14T02:44:03,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/ee523d902a2041749ab8eb897d99526e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/ee523d902a2041749ab8eb897d99526e 2024-11-14T02:44:03,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3b57b33515c24269ae23d7bed94f43bd to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/3b57b33515c24269ae23d7bed94f43bd 2024-11-14T02:44:03,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/59400534e6a44f58abc2647ec7b8498f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/59400534e6a44f58abc2647ec7b8498f 2024-11-14T02:44:03,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/30d1e00f75d14ca7950ac8adb784eef2 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/30d1e00f75d14ca7950ac8adb784eef2 2024-11-14T02:44:03,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/e4ad07f880e54428a1ad90ec7e46f2f1 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/e4ad07f880e54428a1ad90ec7e46f2f1 2024-11-14T02:44:03,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6ff0027c1af74a94a57023087b71d0c6 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/6ff0027c1af74a94a57023087b71d0c6 2024-11-14T02:44:03,989 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/recovered.edits/686.seqid, newMaxSeqId=686, maxSeqId=1 2024-11-14T02:44:03,990 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. 2024-11-14T02:44:03,990 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for b0f2fe4f29c62d92b3ee0b2d165c19cb: Waiting for close lock at 1731552242578Running coprocessor pre-close hooks at 1731552242578Disabling compacts and flushes for region at 1731552242578Disabling writes for close at 1731552242579 (+1 ms)Obtaining lock to block concurrent updates at 1731552242579Preparing flush snapshotting stores in b0f2fe4f29c62d92b3ee0b2d165c19cb at 1731552242579Finished memstore snapshotting TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731552242580 (+1 ms)Flushing stores of TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb. at 1731552242580Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb/A: creating writer at 1731552242580Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb/A: appending metadata at 1731552242584 (+4 ms)Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb/A: closing flushed file at 1731552242584Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb/B: creating writer at 1731552242995 (+411 ms)Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb/B: appending metadata at 1731552242998 (+3 ms)Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb/B: closing flushed file at 1731552242998Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb/C: creating writer at 1731552243411 (+413 ms)Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb/C: appending metadata at 1731552243414 (+3 ms)Flushing b0f2fe4f29c62d92b3ee0b2d165c19cb/C: closing flushed file at 1731552243414Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1475f77a: reopening flushed file at 1731552243823 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ef0adca: reopening flushed file at 1731552243829 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3249b0c7: reopening flushed file at 1731552243834 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for b0f2fe4f29c62d92b3ee0b2d165c19cb in 1263ms, sequenceid=683, compaction requested=true at 1731552243842 (+8 ms)Writing region close event to WAL at 1731552243984 (+142 ms)Running coprocessor post-close hooks at 1731552243990 (+6 ms)Closed at 1731552243990 2024-11-14T02:44:03,993 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:44:03,993 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b0f2fe4f29c62d92b3ee0b2d165c19cb, regionState=CLOSED 2024-11-14T02:44:03,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:44:03,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-14T02:44:03,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure b0f2fe4f29c62d92b3ee0b2d165c19cb, server=e5dc8b5cf5e9,46387,1731552206828 in 1.5770 sec 2024-11-14T02:44:04,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-14T02:44:04,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b0f2fe4f29c62d92b3ee0b2d165c19cb, UNASSIGN in 1.5840 sec 2024-11-14T02:44:04,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-14T02:44:04,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5920 sec 2024-11-14T02:44:04,006 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552244006"}]},"ts":"1731552244006"} 2024-11-14T02:44:04,007 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T02:44:04,008 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T02:44:04,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6100 sec 2024-11-14T02:44:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T02:44:04,556 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:04,558 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T02:44:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:04,563 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T02:44:04,565 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:04,569 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:44:04,574 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/recovered.edits] 2024-11-14T02:44:04,608 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/3332d9bad0e94e31ba80b9599cb9fdb8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/3332d9bad0e94e31ba80b9599cb9fdb8 2024-11-14T02:44:04,610 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/41959445d32a426cb7132f89bb531564 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/41959445d32a426cb7132f89bb531564 2024-11-14T02:44:04,612 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/fcac97f732584d13b9ff7a9e817f5664 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/A/fcac97f732584d13b9ff7a9e817f5664 2024-11-14T02:44:04,614 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/1aa3ae73f30c4fcb8b905a828b8958c2 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/1aa3ae73f30c4fcb8b905a828b8958c2 2024-11-14T02:44:04,616 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/2e90fff25050440a88306ac3b03cfd90 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/2e90fff25050440a88306ac3b03cfd90 2024-11-14T02:44:04,618 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/6fefba764f3242be966bc437aa76d7d2 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/B/6fefba764f3242be966bc437aa76d7d2 2024-11-14T02:44:04,620 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/60f7531360ba48f5a36119a46ca31297 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/60f7531360ba48f5a36119a46ca31297 2024-11-14T02:44:04,622 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/67de34a9312e4f91979ad5343cc02ed0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/67de34a9312e4f91979ad5343cc02ed0 2024-11-14T02:44:04,623 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/d7ded38e6c2f43be82c2d8dd7473a79f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/C/d7ded38e6c2f43be82c2d8dd7473a79f 2024-11-14T02:44:04,626 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/recovered.edits/686.seqid to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb/recovered.edits/686.seqid 2024-11-14T02:44:04,626 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/b0f2fe4f29c62d92b3ee0b2d165c19cb 2024-11-14T02:44:04,626 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T02:44:04,631 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:04,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-14T02:44:04,641 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T02:44:04,671 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T02:44:04,673 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:04,673 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T02:44:04,673 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731552244673"}]},"ts":"9223372036854775807"} 2024-11-14T02:44:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T02:44:04,677 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T02:44:04,677 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b0f2fe4f29c62d92b3ee0b2d165c19cb, NAME => 'TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T02:44:04,677 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T02:44:04,677 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731552244677"}]},"ts":"9223372036854775807"} 2024-11-14T02:44:04,680 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T02:44:04,681 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:04,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 122 msec 2024-11-14T02:44:04,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T02:44:04,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T02:44:04,886 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:04,899 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-521717827_22 at /127.0.0.1:55182 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=383 (was 215) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10077 (was 11046) 2024-11-14T02:44:04,908 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=383, ProcessCount=11, AvailableMemoryMB=10077 2024-11-14T02:44:04,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:44:04,910 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:44:04,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:04,913 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T02:44:04,913 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:04,913 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-14T02:44:04,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-14T02:44:04,915 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T02:44:04,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741941_1117 (size=1061) 2024-11-14T02:44:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-14T02:44:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-14T02:44:05,327 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1 2024-11-14T02:44:05,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741942_1118 (size=53) 2024-11-14T02:44:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-14T02:44:05,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:44:05,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1d50bec2e36af6979b333798612ed279, disabling compactions & flushes 2024-11-14T02:44:05,739 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:05,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:05,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. after waiting 0 ms 2024-11-14T02:44:05,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:05,739 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:05,739 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1d50bec2e36af6979b333798612ed279: Waiting for close lock at 1731552245739Disabling compacts and flushes for region at 1731552245739Disabling writes for close at 1731552245739Writing region close event to WAL at 1731552245739Closed at 1731552245739 2024-11-14T02:44:05,741 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T02:44:05,741 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731552245741"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731552245741"}]},"ts":"1731552245741"} 2024-11-14T02:44:05,744 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T02:44:05,745 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T02:44:05,745 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552245745"}]},"ts":"1731552245745"} 2024-11-14T02:44:05,748 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T02:44:05,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d50bec2e36af6979b333798612ed279, ASSIGN}] 2024-11-14T02:44:05,751 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d50bec2e36af6979b333798612ed279, ASSIGN 2024-11-14T02:44:05,752 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d50bec2e36af6979b333798612ed279, ASSIGN; state=OFFLINE, location=e5dc8b5cf5e9,46387,1731552206828; forceNewPlan=false, retain=false 2024-11-14T02:44:05,902 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1d50bec2e36af6979b333798612ed279, regionState=OPENING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:05,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d50bec2e36af6979b333798612ed279, ASSIGN because future has completed 2024-11-14T02:44:05,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:44:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-14T02:44:06,060 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:06,061 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:44:06,061 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:06,061 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:44:06,061 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:06,061 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:06,063 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:06,064 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:44:06,065 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d50bec2e36af6979b333798612ed279 columnFamilyName A 2024-11-14T02:44:06,065 DEBUG [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:06,066 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.HStore(327): Store=1d50bec2e36af6979b333798612ed279/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:06,066 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:06,067 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:44:06,067 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d50bec2e36af6979b333798612ed279 columnFamilyName B 2024-11-14T02:44:06,067 DEBUG [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:06,068 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.HStore(327): Store=1d50bec2e36af6979b333798612ed279/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:06,068 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:06,069 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:44:06,069 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d50bec2e36af6979b333798612ed279 columnFamilyName C 2024-11-14T02:44:06,070 DEBUG [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:06,070 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.HStore(327): Store=1d50bec2e36af6979b333798612ed279/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:06,070 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:06,070 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:06,071 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:06,074 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:06,075 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:06,075 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:06,076 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:44:06,077 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:06,079 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:44:06,080 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 1d50bec2e36af6979b333798612ed279; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59569879, jitterRate=-0.11233963072299957}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:44:06,080 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:06,081 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 1d50bec2e36af6979b333798612ed279: Running coprocessor pre-open hook at 1731552246061Writing region info on filesystem at 1731552246061Initializing all the Stores at 1731552246062 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552246063 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552246063Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552246063Cleaning up temporary data from old regions at 1731552246075 (+12 ms)Running coprocessor post-open hooks at 1731552246080 (+5 ms)Region opened successfully at 1731552246081 (+1 ms) 2024-11-14T02:44:06,082 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., pid=30, masterSystemTime=1731552246057 2024-11-14T02:44:06,084 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:06,084 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:06,084 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1d50bec2e36af6979b333798612ed279, regionState=OPEN, openSeqNum=2, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:06,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:44:06,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-14T02:44:06,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 in 182 msec 2024-11-14T02:44:06,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-14T02:44:06,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d50bec2e36af6979b333798612ed279, ASSIGN in 341 msec 2024-11-14T02:44:06,093 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T02:44:06,093 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552246093"}]},"ts":"1731552246093"} 2024-11-14T02:44:06,094 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T02:44:06,095 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T02:44:06,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-14T02:44:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-14T02:44:07,066 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:07,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736030eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:07,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:07,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:07,072 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:07,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:07,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:07,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a50264, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:07,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:07,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:07,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:07,075 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:07,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3146dbb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:07,077 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:07,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:07,082 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:44:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:44:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@68ba1929 2024-11-14T02:44:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:44:07,084 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:44:07,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:44:07,089 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:44:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:07,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741943_1119 (size=1097) 2024-11-14T02:44:07,507 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T02:44:07,507 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T02:44:07,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:44:07,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d50bec2e36af6979b333798612ed279, REOPEN/MOVE}] 2024-11-14T02:44:07,518 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d50bec2e36af6979b333798612ed279, REOPEN/MOVE 2024-11-14T02:44:07,518 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=1d50bec2e36af6979b333798612ed279, regionState=CLOSING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:07,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d50bec2e36af6979b333798612ed279, REOPEN/MOVE because future has completed 2024-11-14T02:44:07,521 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:44:07,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:44:07,673 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:07,674 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:44:07,674 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 1d50bec2e36af6979b333798612ed279, disabling compactions & flushes 2024-11-14T02:44:07,674 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:07,674 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:07,674 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. after waiting 0 ms 2024-11-14T02:44:07,674 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:07,680 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T02:44:07,681 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:07,681 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 1d50bec2e36af6979b333798612ed279: Waiting for close lock at 1731552247674Running coprocessor pre-close hooks at 1731552247674Disabling compacts and flushes for region at 1731552247674Disabling writes for close at 1731552247674Writing region close event to WAL at 1731552247675 (+1 ms)Running coprocessor post-close hooks at 1731552247681 (+6 ms)Closed at 1731552247681 2024-11-14T02:44:07,682 WARN [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 1d50bec2e36af6979b333798612ed279 to self. 2024-11-14T02:44:07,684 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:07,685 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=1d50bec2e36af6979b333798612ed279, regionState=CLOSED 2024-11-14T02:44:07,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:44:07,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-14T02:44:07,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 in 168 msec 2024-11-14T02:44:07,692 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d50bec2e36af6979b333798612ed279, REOPEN/MOVE; state=CLOSED, location=e5dc8b5cf5e9,46387,1731552206828; forceNewPlan=false, retain=true 2024-11-14T02:44:07,843 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=1d50bec2e36af6979b333798612ed279, regionState=OPENING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:07,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d50bec2e36af6979b333798612ed279, REOPEN/MOVE because future has completed 2024-11-14T02:44:07,847 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:44:08,007 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:08,008 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:44:08,009 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:08,009 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:44:08,009 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:08,009 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:08,012 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:08,013 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:44:08,018 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d50bec2e36af6979b333798612ed279 columnFamilyName A 2024-11-14T02:44:08,020 DEBUG [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:08,020 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.HStore(327): Store=1d50bec2e36af6979b333798612ed279/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:08,021 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:08,021 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:44:08,022 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d50bec2e36af6979b333798612ed279 columnFamilyName B 2024-11-14T02:44:08,022 DEBUG [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:08,022 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.HStore(327): Store=1d50bec2e36af6979b333798612ed279/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:08,022 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:08,023 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:44:08,023 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d50bec2e36af6979b333798612ed279 columnFamilyName C 2024-11-14T02:44:08,023 DEBUG [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:08,023 INFO [StoreOpener-1d50bec2e36af6979b333798612ed279-1 {}] regionserver.HStore(327): Store=1d50bec2e36af6979b333798612ed279/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:08,024 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:08,024 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:08,024 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:08,026 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:08,027 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:08,027 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:08,027 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:44:08,029 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:08,030 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 1d50bec2e36af6979b333798612ed279; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70274898, jitterRate=0.04717758297920227}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:44:08,030 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:08,032 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 1d50bec2e36af6979b333798612ed279: Running coprocessor pre-open hook at 1731552248010Writing region info on filesystem at 1731552248010Initializing all the Stores at 1731552248011 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552248011Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552248011Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552248011Cleaning up temporary data from old regions at 1731552248027 (+16 ms)Running coprocessor post-open hooks at 1731552248030 (+3 ms)Region opened successfully at 1731552248032 (+2 ms) 2024-11-14T02:44:08,033 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., pid=35, masterSystemTime=1731552248000 2024-11-14T02:44:08,035 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:08,035 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:08,035 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=1d50bec2e36af6979b333798612ed279, regionState=OPEN, openSeqNum=5, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:44:08,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-14T02:44:08,041 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 in 191 msec 2024-11-14T02:44:08,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-14T02:44:08,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d50bec2e36af6979b333798612ed279, REOPEN/MOVE in 524 msec 2024-11-14T02:44:08,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-14T02:44:08,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 535 msec 2024-11-14T02:44:08,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 957 msec 2024-11-14T02:44:08,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-14T02:44:08,052 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:08,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,052 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:08,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:08,053 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1347acde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:08,055 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c69f7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,056 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:08,057 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:08,058 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bae89bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:08,059 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:08,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@345a43b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208b436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:08,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:08,061 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b134dc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:08,063 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d1ecdcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:08,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:08,066 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35907c55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:08,067 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:08,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a8d8ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,068 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:08,068 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:08,069 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61140703, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:08,070 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:08,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@172ab5d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,071 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:08,071 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:08,072 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ce31cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:08,073 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:08,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a27e394, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,074 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:08,074 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:08,075 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35fc5f7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:08,076 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bb5f8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,077 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3806f988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,077 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:08,077 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:08,078 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:08,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:08,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:08,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd929ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:08,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:08,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:08,079 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23a5d2c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494eb83f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,080 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:08,080 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:08,081 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:08,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@291df4e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:08,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:08,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:08,083 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:08,084 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eacef77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:08,084 DEBUG [Thread-608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:08,084 DEBUG [Thread-623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:08,084 DEBUG [Thread-621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:08,084 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:08,085 DEBUG [Thread-619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:08,085 DEBUG [Thread-616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:08,086 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:08,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:08,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:08,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:08,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:08,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:08,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:08,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:08,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:08,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:08,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:08,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:08,088 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:08,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:08,089 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:08,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:08,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:08,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:08,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:08,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:08,091 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:44:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:08,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:44:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:44:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:44:08,095 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:44:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:44:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:44:08,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:44:08,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:44:08,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-14T02:44:08,103 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T02:44:08,108 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:08,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:08,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:08,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:44:08,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:08,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:08,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:08,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:08,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:08,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:08,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146a84d8eb1a51408593fd9609454ab20b_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552248110/Put/seqid=0 2024-11-14T02:44:08,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741944_1120 (size=12154) 2024-11-14T02:44:08,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731552248177, max timestamp=1731552248197Num uniques 859; 2024-11-14T02:44:08,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731552248177, max timestamp=1731552248197Num uniques 859; 2024-11-14T02:44:08,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T02:44:08,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T02:44:08,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731552248177, max timestamp=1731552248197Num uniques 859; 2024-11-14T02:44:08,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T02:44:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T02:44:08,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552308227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552308232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552308241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552308234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552308247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T02:44:08,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:08,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:08,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:08,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:08,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:08,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552308346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552308346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552308355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552308355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552308355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T02:44:08,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T02:44:08,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:08,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:08,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:08,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:08,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:08,556 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:08,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552308556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552308556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,562 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146a84d8eb1a51408593fd9609454ab20b_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146a84d8eb1a51408593fd9609454ab20b_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:08,563 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d309599af204450f9c5475482328664f, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:08,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552308565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552308565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552308565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d309599af204450f9c5475482328664f is 175, key is test_row_0/A:col10/1731552248110/Put/seqid=0 2024-11-14T02:44:08,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T02:44:08,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:08,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:08,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:08,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:08,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:08,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741945_1121 (size=30955) 2024-11-14T02:44:08,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T02:44:08,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:08,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:08,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:08,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:08,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T02:44:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:08,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552308865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552308865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552308876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552308876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552308876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:08,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T02:44:08,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:08,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:08,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:08,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:08,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:09,043 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d309599af204450f9c5475482328664f 2024-11-14T02:44:09,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T02:44:09,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:09,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:09,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:09,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:09,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:09,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/e03fdd9f02744cb1bd74590c43d9972e is 50, key is test_row_0/B:col10/1731552248110/Put/seqid=0 2024-11-14T02:44:09,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741946_1122 (size=12001) 2024-11-14T02:44:09,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/e03fdd9f02744cb1bd74590c43d9972e 2024-11-14T02:44:09,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/d8398b46e3304b3b826332880b069750 is 50, key is test_row_0/C:col10/1731552248110/Put/seqid=0 2024-11-14T02:44:09,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741947_1123 (size=12001) 2024-11-14T02:44:09,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/d8398b46e3304b3b826332880b069750 2024-11-14T02:44:09,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d309599af204450f9c5475482328664f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d309599af204450f9c5475482328664f 2024-11-14T02:44:09,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d309599af204450f9c5475482328664f, entries=150, sequenceid=15, filesize=30.2 K 2024-11-14T02:44:09,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/e03fdd9f02744cb1bd74590c43d9972e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e03fdd9f02744cb1bd74590c43d9972e 2024-11-14T02:44:09,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e03fdd9f02744cb1bd74590c43d9972e, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T02:44:09,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/d8398b46e3304b3b826332880b069750 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/d8398b46e3304b3b826332880b069750 2024-11-14T02:44:09,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T02:44:09,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:09,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:09,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:09,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:09,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:09,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/d8398b46e3304b3b826332880b069750, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T02:44:09,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for 1d50bec2e36af6979b333798612ed279 in 1097ms, sequenceid=15, compaction requested=false 2024-11-14T02:44:09,207 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T02:44:09,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T02:44:09,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T02:44:09,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:09,357 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-14T02:44:09,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:09,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143f94dddb45944faa83e32edeb616aa46_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552248240/Put/seqid=0 2024-11-14T02:44:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:09,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:09,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741948_1124 (size=12154) 2024-11-14T02:44:09,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:09,386 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143f94dddb45944faa83e32edeb616aa46_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143f94dddb45944faa83e32edeb616aa46_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:09,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/696c8501f31c41638b164b282ddaceff, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:09,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/696c8501f31c41638b164b282ddaceff is 175, key is test_row_0/A:col10/1731552248240/Put/seqid=0 2024-11-14T02:44:09,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552309391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552309393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741949_1125 (size=30955) 2024-11-14T02:44:09,402 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/696c8501f31c41638b164b282ddaceff 2024-11-14T02:44:09,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552309395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552309395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552309397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/e947872ac8764563b2c4a0dc978ba122 is 50, key is test_row_0/B:col10/1731552248240/Put/seqid=0 2024-11-14T02:44:09,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741950_1126 (size=12001) 2024-11-14T02:44:09,428 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/e947872ac8764563b2c4a0dc978ba122 2024-11-14T02:44:09,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/78cd95cd3476456788f02d56e2817cb0 is 50, key is test_row_0/C:col10/1731552248240/Put/seqid=0 2024-11-14T02:44:09,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741951_1127 (size=12001) 2024-11-14T02:44:09,468 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/78cd95cd3476456788f02d56e2817cb0 2024-11-14T02:44:09,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/696c8501f31c41638b164b282ddaceff as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/696c8501f31c41638b164b282ddaceff 2024-11-14T02:44:09,487 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/696c8501f31c41638b164b282ddaceff, entries=150, sequenceid=50, filesize=30.2 K 2024-11-14T02:44:09,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/e947872ac8764563b2c4a0dc978ba122 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e947872ac8764563b2c4a0dc978ba122 2024-11-14T02:44:09,494 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e947872ac8764563b2c4a0dc978ba122, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T02:44:09,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/78cd95cd3476456788f02d56e2817cb0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/78cd95cd3476456788f02d56e2817cb0 2024-11-14T02:44:09,506 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/78cd95cd3476456788f02d56e2817cb0, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T02:44:09,507 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=53.67 KB/54960 for 1d50bec2e36af6979b333798612ed279 in 150ms, sequenceid=50, compaction requested=false 2024-11-14T02:44:09,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:09,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:09,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-14T02:44:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-14T02:44:09,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T02:44:09,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:09,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:09,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:09,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:09,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:09,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:09,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-14T02:44:09,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-11-14T02:44:09,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.4120 sec 2024-11-14T02:44:09,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144418a2dc9d4c4ee984af0eb5f2ead2bb_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552249391/Put/seqid=0 2024-11-14T02:44:09,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741952_1128 (size=14594) 2024-11-14T02:44:09,543 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T02:44:09,549 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:09,556 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144418a2dc9d4c4ee984af0eb5f2ead2bb_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144418a2dc9d4c4ee984af0eb5f2ead2bb_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:09,558 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d9c096da6cfb4ab6ae045de6c8357284, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:09,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d9c096da6cfb4ab6ae045de6c8357284 is 175, key is test_row_0/A:col10/1731552249391/Put/seqid=0 2024-11-14T02:44:09,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552309549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552309549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552309550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552309553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741953_1129 (size=39549) 2024-11-14T02:44:09,615 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d9c096da6cfb4ab6ae045de6c8357284 2024-11-14T02:44:09,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/b5813955a23e48f5905e5114a1914db3 is 50, key is test_row_0/B:col10/1731552249391/Put/seqid=0 2024-11-14T02:44:09,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741954_1130 (size=12001) 2024-11-14T02:44:09,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/b5813955a23e48f5905e5114a1914db3 2024-11-14T02:44:09,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/00aff73f35f140088b7f02da9a1594b9 is 50, key is test_row_0/C:col10/1731552249391/Put/seqid=0 2024-11-14T02:44:09,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741955_1131 (size=12001) 2024-11-14T02:44:09,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/00aff73f35f140088b7f02da9a1594b9 2024-11-14T02:44:09,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552309666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552309666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552309666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552309666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d9c096da6cfb4ab6ae045de6c8357284 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d9c096da6cfb4ab6ae045de6c8357284 2024-11-14T02:44:09,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d9c096da6cfb4ab6ae045de6c8357284, entries=200, sequenceid=62, filesize=38.6 K 2024-11-14T02:44:09,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/b5813955a23e48f5905e5114a1914db3 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/b5813955a23e48f5905e5114a1914db3 2024-11-14T02:44:09,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/b5813955a23e48f5905e5114a1914db3, entries=150, sequenceid=62, filesize=11.7 K 2024-11-14T02:44:09,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/00aff73f35f140088b7f02da9a1594b9 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/00aff73f35f140088b7f02da9a1594b9 2024-11-14T02:44:09,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/00aff73f35f140088b7f02da9a1594b9, entries=150, sequenceid=62, filesize=11.7 K 2024-11-14T02:44:09,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1d50bec2e36af6979b333798612ed279 in 199ms, sequenceid=62, compaction requested=true 2024-11-14T02:44:09,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:09,709 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:09,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:09,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:09,710 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:09,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:09,711 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:09,711 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/A is initiating minor compaction (all files) 2024-11-14T02:44:09,711 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/A in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:09,711 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d309599af204450f9c5475482328664f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/696c8501f31c41638b164b282ddaceff, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d9c096da6cfb4ab6ae045de6c8357284] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=99.1 K 2024-11-14T02:44:09,711 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:09,712 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d309599af204450f9c5475482328664f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/696c8501f31c41638b164b282ddaceff, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d9c096da6cfb4ab6ae045de6c8357284] 2024-11-14T02:44:09,713 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting d309599af204450f9c5475482328664f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731552248103 2024-11-14T02:44:09,714 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:09,714 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/B is initiating minor compaction (all files) 2024-11-14T02:44:09,714 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/B in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:09,714 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e03fdd9f02744cb1bd74590c43d9972e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e947872ac8764563b2c4a0dc978ba122, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/b5813955a23e48f5905e5114a1914db3] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=35.2 K 2024-11-14T02:44:09,717 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 696c8501f31c41638b164b282ddaceff, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731552248223 2024-11-14T02:44:09,719 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting e03fdd9f02744cb1bd74590c43d9972e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731552248103 2024-11-14T02:44:09,720 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9c096da6cfb4ab6ae045de6c8357284, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731552249390 2024-11-14T02:44:09,720 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting e947872ac8764563b2c4a0dc978ba122, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731552248223 2024-11-14T02:44:09,720 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting b5813955a23e48f5905e5114a1914db3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731552249390 2024-11-14T02:44:09,736 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#B#compaction#113 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:09,737 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/1b9f9e3ff12249dbb9d1a9bbd36abb88 is 50, key is test_row_0/B:col10/1731552249391/Put/seqid=0 2024-11-14T02:44:09,745 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:09,757 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411147024d9a7cbaf47ff9bd10468ac07a7dc_1d50bec2e36af6979b333798612ed279 store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:09,765 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411147024d9a7cbaf47ff9bd10468ac07a7dc_1d50bec2e36af6979b333798612ed279, store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:09,766 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147024d9a7cbaf47ff9bd10468ac07a7dc_1d50bec2e36af6979b333798612ed279 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:09,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741956_1132 (size=12104) 2024-11-14T02:44:09,798 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/1b9f9e3ff12249dbb9d1a9bbd36abb88 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/1b9f9e3ff12249dbb9d1a9bbd36abb88 2024-11-14T02:44:09,806 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/B of 1d50bec2e36af6979b333798612ed279 into 1b9f9e3ff12249dbb9d1a9bbd36abb88(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:09,806 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:09,806 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/B, priority=13, startTime=1731552249709; duration=0sec 2024-11-14T02:44:09,806 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:09,806 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:B 2024-11-14T02:44:09,806 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:09,809 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:09,809 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/C is initiating minor compaction (all files) 2024-11-14T02:44:09,809 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/C in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:09,809 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/d8398b46e3304b3b826332880b069750, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/78cd95cd3476456788f02d56e2817cb0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/00aff73f35f140088b7f02da9a1594b9] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=35.2 K 2024-11-14T02:44:09,810 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting d8398b46e3304b3b826332880b069750, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731552248103 2024-11-14T02:44:09,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741957_1133 (size=4469) 2024-11-14T02:44:09,812 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 78cd95cd3476456788f02d56e2817cb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731552248223 2024-11-14T02:44:09,813 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 00aff73f35f140088b7f02da9a1594b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731552249390 2024-11-14T02:44:09,813 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#A#compaction#114 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:09,814 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/b9c12bcc27f94717a4f0aa736b4cb7c6 is 175, key is test_row_0/A:col10/1731552249391/Put/seqid=0 2024-11-14T02:44:09,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741958_1134 (size=31058) 2024-11-14T02:44:09,843 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#C#compaction#115 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:09,844 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/b6c6bb1fb1334fe2b1a986fa2064a91a is 50, key is test_row_0/C:col10/1731552249391/Put/seqid=0 2024-11-14T02:44:09,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741959_1135 (size=12104) 2024-11-14T02:44:09,864 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/b6c6bb1fb1334fe2b1a986fa2064a91a as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/b6c6bb1fb1334fe2b1a986fa2064a91a 2024-11-14T02:44:09,875 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/C of 1d50bec2e36af6979b333798612ed279 into b6c6bb1fb1334fe2b1a986fa2064a91a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:09,875 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:09,875 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/C, priority=13, startTime=1731552249710; duration=0sec 2024-11-14T02:44:09,876 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:09,876 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:C 2024-11-14T02:44:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:09,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T02:44:09,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:09,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:09,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:09,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:09,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:09,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:09,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c243b880a4714fb08a40742915017cb1_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552249550/Put/seqid=0 2024-11-14T02:44:09,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552309900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552309907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552309909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552309910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:09,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:09,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741960_1136 (size=14594) 2024-11-14T02:44:09,923 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:09,930 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c243b880a4714fb08a40742915017cb1_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c243b880a4714fb08a40742915017cb1_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:09,931 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/1185202aa0e348649e054ed991f7d727, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:09,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/1185202aa0e348649e054ed991f7d727 is 175, key is test_row_0/A:col10/1731552249550/Put/seqid=0 2024-11-14T02:44:09,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741961_1137 (size=39549) 2024-11-14T02:44:09,949 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/1185202aa0e348649e054ed991f7d727 2024-11-14T02:44:09,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/b85396eee9264722afa794e430e8cdfb is 50, key is test_row_0/B:col10/1731552249550/Put/seqid=0 2024-11-14T02:44:09,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741962_1138 (size=12001) 2024-11-14T02:44:09,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/b85396eee9264722afa794e430e8cdfb 2024-11-14T02:44:10,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/e3e87287c951440e95763bad46839de8 is 50, key is test_row_0/C:col10/1731552249550/Put/seqid=0 2024-11-14T02:44:10,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741963_1139 (size=12001) 2024-11-14T02:44:10,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552310015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552310015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552310025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552310025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552310225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552310225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552310236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552310236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,247 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/b9c12bcc27f94717a4f0aa736b4cb7c6 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/b9c12bcc27f94717a4f0aa736b4cb7c6 2024-11-14T02:44:10,254 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/A of 1d50bec2e36af6979b333798612ed279 into b9c12bcc27f94717a4f0aa736b4cb7c6(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:10,254 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:10,254 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/A, priority=13, startTime=1731552249709; duration=0sec 2024-11-14T02:44:10,254 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:10,254 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:A 2024-11-14T02:44:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T02:44:10,255 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:10,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-14T02:44:10,259 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T02:44:10,260 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:10,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:10,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T02:44:10,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/e3e87287c951440e95763bad46839de8 2024-11-14T02:44:10,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T02:44:10,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:10,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:10,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:10,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:10,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:10,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552310416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/1185202aa0e348649e054ed991f7d727 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/1185202aa0e348649e054ed991f7d727 2024-11-14T02:44:10,417 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in 1d50bec2e36af6979b333798612ed279 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/1185202aa0e348649e054ed991f7d727, entries=200, sequenceid=85, filesize=38.6 K 2024-11-14T02:44:10,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/b85396eee9264722afa794e430e8cdfb as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/b85396eee9264722afa794e430e8cdfb 2024-11-14T02:44:10,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/b85396eee9264722afa794e430e8cdfb, entries=150, sequenceid=85, filesize=11.7 K 2024-11-14T02:44:10,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/e3e87287c951440e95763bad46839de8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/e3e87287c951440e95763bad46839de8 2024-11-14T02:44:10,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/e3e87287c951440e95763bad46839de8, entries=150, sequenceid=85, filesize=11.7 K 2024-11-14T02:44:10,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 1d50bec2e36af6979b333798612ed279 in 564ms, sequenceid=85, compaction requested=false 2024-11-14T02:44:10,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:10,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T02:44:10,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:10,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:10,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:10,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:10,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:10,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:10,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ed247f869dde4fb585aae51f190e0267_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552249903/Put/seqid=0 2024-11-14T02:44:10,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T02:44:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:10,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T02:44:10,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552310571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552310572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552310573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552310578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741964_1140 (size=14594) 2024-11-14T02:44:10,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552310685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552310686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552310686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552310686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T02:44:10,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:10,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:10,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:10,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:10,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:10,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T02:44:10,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:10,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:10,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:10,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:10,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T02:44:10,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552310896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552310896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552310896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552310896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:10,999 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:11,004 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ed247f869dde4fb585aae51f190e0267_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ed247f869dde4fb585aae51f190e0267_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:11,005 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/c46ddbbdfc6d440a8bbf5f3fcd9e51cd, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:11,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/c46ddbbdfc6d440a8bbf5f3fcd9e51cd is 175, key is test_row_0/A:col10/1731552249903/Put/seqid=0 2024-11-14T02:44:11,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741965_1141 (size=39549) 2024-11-14T02:44:11,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T02:44:11,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:11,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:11,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:11,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T02:44:11,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:11,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:11,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:11,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552311205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:11,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:11,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552311205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:11,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552311205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552311206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:11,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T02:44:11,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:11,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:11,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:11,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T02:44:11,417 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/c46ddbbdfc6d440a8bbf5f3fcd9e51cd 2024-11-14T02:44:11,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/100653382260487e9aac89d7df73080a is 50, key is test_row_0/B:col10/1731552249903/Put/seqid=0 2024-11-14T02:44:11,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741966_1142 (size=12001) 2024-11-14T02:44:11,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T02:44:11,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:11,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:11,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:11,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,490 INFO [master/e5dc8b5cf5e9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T02:44:11,490 INFO [master/e5dc8b5cf5e9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T02:44:11,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T02:44:11,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:11,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:11,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:11,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552311715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:11,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:11,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552311715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:11,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552311725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:11,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:11,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552311725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:11,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T02:44:11,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:11,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:11,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:11,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/100653382260487e9aac89d7df73080a 2024-11-14T02:44:11,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/4fd6f721b1af49a092ba68b48b596a50 is 50, key is test_row_0/C:col10/1731552249903/Put/seqid=0 2024-11-14T02:44:11,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741967_1143 (size=12001) 2024-11-14T02:44:11,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T02:44:11,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:11,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:11,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:11,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:12,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T02:44:12,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:12,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:12,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:12,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:12,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:12,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/4fd6f721b1af49a092ba68b48b596a50 2024-11-14T02:44:12,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/c46ddbbdfc6d440a8bbf5f3fcd9e51cd as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/c46ddbbdfc6d440a8bbf5f3fcd9e51cd 2024-11-14T02:44:12,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T02:44:12,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:12,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:12,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:12,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:12,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:12,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/c46ddbbdfc6d440a8bbf5f3fcd9e51cd, entries=200, sequenceid=99, filesize=38.6 K 2024-11-14T02:44:12,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/100653382260487e9aac89d7df73080a as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/100653382260487e9aac89d7df73080a 2024-11-14T02:44:12,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/100653382260487e9aac89d7df73080a, entries=150, sequenceid=99, filesize=11.7 K 2024-11-14T02:44:12,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/4fd6f721b1af49a092ba68b48b596a50 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/4fd6f721b1af49a092ba68b48b596a50 2024-11-14T02:44:12,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/4fd6f721b1af49a092ba68b48b596a50, entries=150, sequenceid=99, filesize=11.7 K 2024-11-14T02:44:12,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1d50bec2e36af6979b333798612ed279 in 1736ms, sequenceid=99, compaction requested=true 2024-11-14T02:44:12,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:12,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:12,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:12,276 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:12,276 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:12,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:12,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:12,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:12,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:12,277 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:12,277 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:12,277 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/B is initiating minor compaction (all files) 2024-11-14T02:44:12,277 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/A is initiating minor compaction (all files) 2024-11-14T02:44:12,278 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/B in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:12,278 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/A in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:12,278 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/b9c12bcc27f94717a4f0aa736b4cb7c6, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/1185202aa0e348649e054ed991f7d727, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/c46ddbbdfc6d440a8bbf5f3fcd9e51cd] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=107.6 K 2024-11-14T02:44:12,278 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/1b9f9e3ff12249dbb9d1a9bbd36abb88, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/b85396eee9264722afa794e430e8cdfb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/100653382260487e9aac89d7df73080a] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=35.3 K 2024-11-14T02:44:12,278 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:12,278 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/b9c12bcc27f94717a4f0aa736b4cb7c6, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/1185202aa0e348649e054ed991f7d727, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/c46ddbbdfc6d440a8bbf5f3fcd9e51cd] 2024-11-14T02:44:12,278 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b9f9e3ff12249dbb9d1a9bbd36abb88, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731552249390 2024-11-14T02:44:12,278 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9c12bcc27f94717a4f0aa736b4cb7c6, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731552249390 2024-11-14T02:44:12,280 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting b85396eee9264722afa794e430e8cdfb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731552249545 2024-11-14T02:44:12,280 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1185202aa0e348649e054ed991f7d727, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731552249536 2024-11-14T02:44:12,281 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting c46ddbbdfc6d440a8bbf5f3fcd9e51cd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731552249892 2024-11-14T02:44:12,281 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 100653382260487e9aac89d7df73080a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731552249892 2024-11-14T02:44:12,288 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:12,293 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:12,293 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/23720e9a0f1c40c886b5c0dd7556f5db is 50, key is test_row_0/B:col10/1731552249903/Put/seqid=0 2024-11-14T02:44:12,297 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111491cacfb3313f4d39afbabee01cc425a8_1d50bec2e36af6979b333798612ed279 store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:12,300 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111491cacfb3313f4d39afbabee01cc425a8_1d50bec2e36af6979b333798612ed279, store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:12,301 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111491cacfb3313f4d39afbabee01cc425a8_1d50bec2e36af6979b333798612ed279 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:12,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741968_1144 (size=12207) 2024-11-14T02:44:12,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741969_1145 (size=4469) 2024-11-14T02:44:12,321 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/23720e9a0f1c40c886b5c0dd7556f5db as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/23720e9a0f1c40c886b5c0dd7556f5db 2024-11-14T02:44:12,322 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#A#compaction#122 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:12,323 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d436d32a42af49429b430297f25f2cf0 is 175, key is test_row_0/A:col10/1731552249903/Put/seqid=0 2024-11-14T02:44:12,334 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/B of 1d50bec2e36af6979b333798612ed279 into 23720e9a0f1c40c886b5c0dd7556f5db(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:12,334 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:12,334 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/B, priority=13, startTime=1731552252276; duration=0sec 2024-11-14T02:44:12,334 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:12,334 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:B 2024-11-14T02:44:12,334 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:12,335 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:12,335 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/C is initiating minor compaction (all files) 2024-11-14T02:44:12,335 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/C in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:12,336 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/b6c6bb1fb1334fe2b1a986fa2064a91a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/e3e87287c951440e95763bad46839de8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/4fd6f721b1af49a092ba68b48b596a50] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=35.3 K 2024-11-14T02:44:12,336 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting b6c6bb1fb1334fe2b1a986fa2064a91a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731552249390 2024-11-14T02:44:12,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741970_1146 (size=31161) 2024-11-14T02:44:12,338 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting e3e87287c951440e95763bad46839de8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731552249545 2024-11-14T02:44:12,338 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fd6f721b1af49a092ba68b48b596a50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731552249892 2024-11-14T02:44:12,345 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d436d32a42af49429b430297f25f2cf0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d436d32a42af49429b430297f25f2cf0 2024-11-14T02:44:12,347 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#C#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:12,347 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/3ba7801d213a4576b15cb09fb4a5fd22 is 50, key is test_row_0/C:col10/1731552249903/Put/seqid=0 2024-11-14T02:44:12,351 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/A of 1d50bec2e36af6979b333798612ed279 into d436d32a42af49429b430297f25f2cf0(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:12,352 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:12,352 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/A, priority=13, startTime=1731552252276; duration=0sec 2024-11-14T02:44:12,352 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:12,352 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:A 2024-11-14T02:44:12,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741971_1147 (size=12207) 2024-11-14T02:44:12,372 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/3ba7801d213a4576b15cb09fb4a5fd22 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/3ba7801d213a4576b15cb09fb4a5fd22 2024-11-14T02:44:12,387 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/C of 1d50bec2e36af6979b333798612ed279 into 3ba7801d213a4576b15cb09fb4a5fd22(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:12,387 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:12,387 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/C, priority=13, startTime=1731552252277; duration=0sec 2024-11-14T02:44:12,387 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:12,387 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:C 2024-11-14T02:44:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T02:44:12,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T02:44:12,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:12,423 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T02:44:12,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:12,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:12,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:12,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:12,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:12,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:12,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f425e1979d1e46b7a7c28341e3aaf4d7_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552250571/Put/seqid=0 2024-11-14T02:44:12,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:12,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741972_1148 (size=12154) 2024-11-14T02:44:12,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552312471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:12,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552312585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:12,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552312725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:12,726 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_2' in 1d50bec2e36af6979b333798612ed279 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:12,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552312735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:12,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552312735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:12,740 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_0' in 1d50bec2e36af6979b333798612ed279 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:12,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:12,740 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in 1d50bec2e36af6979b333798612ed279 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:12,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:12,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:12,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:12,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552312745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:12,750 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in 1d50bec2e36af6979b333798612ed279 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:12,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552312795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:12,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:12,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:12,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:12,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:12,857 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f425e1979d1e46b7a7c28341e3aaf4d7_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f425e1979d1e46b7a7c28341e3aaf4d7_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:12,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/1a5ac89bd6584dde9b661b02bb3e47d0, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:12,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/1a5ac89bd6584dde9b661b02bb3e47d0 is 175, key is test_row_0/A:col10/1731552250571/Put/seqid=0 2024-11-14T02:44:12,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741973_1149 (size=30955) 2024-11-14T02:44:12,864 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/1a5ac89bd6584dde9b661b02bb3e47d0 2024-11-14T02:44:12,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/efcbd1b177cb4825bbd8c8c5f394420d is 50, key is test_row_0/B:col10/1731552250571/Put/seqid=0 2024-11-14T02:44:12,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741974_1150 (size=12001) 2024-11-14T02:44:13,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:13,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552313105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:13,286 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/efcbd1b177cb4825bbd8c8c5f394420d 2024-11-14T02:44:13,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/8c2521161aa04a26b1d116a98751b9db is 50, key is test_row_0/C:col10/1731552250571/Put/seqid=0 2024-11-14T02:44:13,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741975_1151 (size=12001) 2024-11-14T02:44:13,302 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/8c2521161aa04a26b1d116a98751b9db 2024-11-14T02:44:13,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/1a5ac89bd6584dde9b661b02bb3e47d0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/1a5ac89bd6584dde9b661b02bb3e47d0 2024-11-14T02:44:13,316 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/1a5ac89bd6584dde9b661b02bb3e47d0, entries=150, sequenceid=121, filesize=30.2 K 2024-11-14T02:44:13,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-14T02:44:13,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/efcbd1b177cb4825bbd8c8c5f394420d as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/efcbd1b177cb4825bbd8c8c5f394420d 2024-11-14T02:44:13,321 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/efcbd1b177cb4825bbd8c8c5f394420d, entries=150, sequenceid=121, filesize=11.7 K 2024-11-14T02:44:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/8c2521161aa04a26b1d116a98751b9db as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/8c2521161aa04a26b1d116a98751b9db 2024-11-14T02:44:13,330 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/8c2521161aa04a26b1d116a98751b9db, entries=150, sequenceid=121, filesize=11.7 K 2024-11-14T02:44:13,331 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1d50bec2e36af6979b333798612ed279 in 907ms, sequenceid=121, compaction requested=false 2024-11-14T02:44:13,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:13,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:13,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-14T02:44:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-14T02:44:13,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-14T02:44:13,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0730 sec 2024-11-14T02:44:13,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 3.0840 sec 2024-11-14T02:44:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:13,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T02:44:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:13,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149be1a3758b2c461697ce22ae30f40cd0_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552252463/Put/seqid=0 2024-11-14T02:44:13,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741976_1152 (size=12304) 2024-11-14T02:44:13,635 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:13,641 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149be1a3758b2c461697ce22ae30f40cd0_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149be1a3758b2c461697ce22ae30f40cd0_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:13,642 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/ac75e8708bd04df1bb4f5518640be40b, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:13,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/ac75e8708bd04df1bb4f5518640be40b is 175, key is test_row_0/A:col10/1731552252463/Put/seqid=0 2024-11-14T02:44:13,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741977_1153 (size=31105) 2024-11-14T02:44:13,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552313688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:13,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:13,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552313795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:14,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552314005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:14,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:14,052 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/ac75e8708bd04df1bb4f5518640be40b 2024-11-14T02:44:14,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/8f869c58e0d64a52af521812457fa721 is 50, key is test_row_0/B:col10/1731552252463/Put/seqid=0 2024-11-14T02:44:14,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741978_1154 (size=12151) 2024-11-14T02:44:14,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552314315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T02:44:14,425 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:14,426 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-14T02:44:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T02:44:14,427 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:14,428 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:14,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:14,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/8f869c58e0d64a52af521812457fa721 2024-11-14T02:44:14,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/f4eaab2b00484a6193d8721220a41884 is 50, key is test_row_0/C:col10/1731552252463/Put/seqid=0 2024-11-14T02:44:14,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741979_1155 (size=12151) 2024-11-14T02:44:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T02:44:14,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:44:14,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:14,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:14,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:14,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:44:14,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:14,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:14,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:14,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T02:44:14,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552314745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:14,748 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_0' in 1d50bec2e36af6979b333798612ed279 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:14,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:14,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552314755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:14,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552314755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:14,759 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_2' in 1d50bec2e36af6979b333798612ed279 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:14,759 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in 1d50bec2e36af6979b333798612ed279 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:14,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552314775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:14,776 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in 1d50bec2e36af6979b333798612ed279 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:14,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:14,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552314825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:14,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/f4eaab2b00484a6193d8721220a41884 2024-11-14T02:44:14,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/ac75e8708bd04df1bb4f5518640be40b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/ac75e8708bd04df1bb4f5518640be40b 2024-11-14T02:44:14,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:44:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:14,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:14,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/ac75e8708bd04df1bb4f5518640be40b, entries=150, sequenceid=135, filesize=30.4 K 2024-11-14T02:44:14,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/8f869c58e0d64a52af521812457fa721 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/8f869c58e0d64a52af521812457fa721 2024-11-14T02:44:14,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/8f869c58e0d64a52af521812457fa721, entries=150, sequenceid=135, filesize=11.9 K 2024-11-14T02:44:14,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/f4eaab2b00484a6193d8721220a41884 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/f4eaab2b00484a6193d8721220a41884 2024-11-14T02:44:14,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/f4eaab2b00484a6193d8721220a41884, entries=150, sequenceid=135, filesize=11.9 K 2024-11-14T02:44:14,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1d50bec2e36af6979b333798612ed279 in 1283ms, sequenceid=135, compaction requested=true 2024-11-14T02:44:14,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:14,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:14,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:14,901 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:14,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:14,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:14,901 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:14,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:14,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:14,902 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:14,902 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/A is initiating minor compaction (all files) 2024-11-14T02:44:14,902 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/A in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:14,902 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d436d32a42af49429b430297f25f2cf0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/1a5ac89bd6584dde9b661b02bb3e47d0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/ac75e8708bd04df1bb4f5518640be40b] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=91.0 K 2024-11-14T02:44:14,902 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:14,902 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d436d32a42af49429b430297f25f2cf0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/1a5ac89bd6584dde9b661b02bb3e47d0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/ac75e8708bd04df1bb4f5518640be40b] 2024-11-14T02:44:14,903 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting d436d32a42af49429b430297f25f2cf0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731552249892 2024-11-14T02:44:14,903 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a5ac89bd6584dde9b661b02bb3e47d0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731552250570 2024-11-14T02:44:14,903 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:14,903 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/B is initiating minor compaction (all files) 2024-11-14T02:44:14,903 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/B in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:14,903 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/23720e9a0f1c40c886b5c0dd7556f5db, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/efcbd1b177cb4825bbd8c8c5f394420d, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/8f869c58e0d64a52af521812457fa721] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=35.5 K 2024-11-14T02:44:14,903 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac75e8708bd04df1bb4f5518640be40b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731552252463 2024-11-14T02:44:14,904 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 23720e9a0f1c40c886b5c0dd7556f5db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731552249892 2024-11-14T02:44:14,904 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting efcbd1b177cb4825bbd8c8c5f394420d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731552250570 2024-11-14T02:44:14,904 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f869c58e0d64a52af521812457fa721, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731552252463 2024-11-14T02:44:14,910 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:14,912 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e518a2bb09304bbda3e63d009ebbdeca_1d50bec2e36af6979b333798612ed279 store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:14,913 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#B#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:14,913 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/0695495ac02249e382be884da73f71b3 is 50, key is test_row_0/B:col10/1731552252463/Put/seqid=0 2024-11-14T02:44:14,915 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e518a2bb09304bbda3e63d009ebbdeca_1d50bec2e36af6979b333798612ed279, store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:14,915 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e518a2bb09304bbda3e63d009ebbdeca_1d50bec2e36af6979b333798612ed279 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:14,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741980_1156 (size=12459) 2024-11-14T02:44:14,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741981_1157 (size=4469) 2024-11-14T02:44:14,921 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#A#compaction#131 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:14,921 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d06b702d67c34117aae5fcdc6dd560a8 is 175, key is test_row_0/A:col10/1731552252463/Put/seqid=0 2024-11-14T02:44:14,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741982_1158 (size=31413) 2024-11-14T02:44:15,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T02:44:15,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:15,040 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T02:44:15,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:15,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:15,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:15,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:15,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:15,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:15,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c4d010cbb6f74aa49484a0972050f0f6_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552253675/Put/seqid=0 2024-11-14T02:44:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T02:44:15,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741983_1159 (size=12304) 2024-11-14T02:44:15,325 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/0695495ac02249e382be884da73f71b3 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/0695495ac02249e382be884da73f71b3 2024-11-14T02:44:15,341 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d06b702d67c34117aae5fcdc6dd560a8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d06b702d67c34117aae5fcdc6dd560a8 2024-11-14T02:44:15,341 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/B of 1d50bec2e36af6979b333798612ed279 into 0695495ac02249e382be884da73f71b3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:15,341 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:15,341 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/B, priority=13, startTime=1731552254901; duration=0sec 2024-11-14T02:44:15,341 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:15,341 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:B 2024-11-14T02:44:15,341 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:15,344 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:15,344 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/C is initiating minor compaction (all files) 2024-11-14T02:44:15,344 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/C in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:15,344 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/3ba7801d213a4576b15cb09fb4a5fd22, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/8c2521161aa04a26b1d116a98751b9db, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/f4eaab2b00484a6193d8721220a41884] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=35.5 K 2024-11-14T02:44:15,345 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ba7801d213a4576b15cb09fb4a5fd22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731552249892 2024-11-14T02:44:15,346 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c2521161aa04a26b1d116a98751b9db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731552250570 2024-11-14T02:44:15,346 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting f4eaab2b00484a6193d8721220a41884, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731552252463 2024-11-14T02:44:15,350 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/A of 1d50bec2e36af6979b333798612ed279 into d06b702d67c34117aae5fcdc6dd560a8(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:15,350 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:15,350 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/A, priority=13, startTime=1731552254901; duration=0sec 2024-11-14T02:44:15,350 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:15,350 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:A 2024-11-14T02:44:15,358 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#C#compaction#134 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:15,359 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/4f98bfe7adfe4f56bedd968d12470421 is 50, key is test_row_0/C:col10/1731552252463/Put/seqid=0 2024-11-14T02:44:15,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741984_1160 (size=12459) 2024-11-14T02:44:15,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:15,471 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c4d010cbb6f74aa49484a0972050f0f6_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c4d010cbb6f74aa49484a0972050f0f6_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:15,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/33404f14b4774e11987b69e7901ff457, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:15,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/33404f14b4774e11987b69e7901ff457 is 175, key is test_row_0/A:col10/1731552253675/Put/seqid=0 2024-11-14T02:44:15,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741985_1161 (size=31105) 2024-11-14T02:44:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T02:44:15,772 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/4f98bfe7adfe4f56bedd968d12470421 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/4f98bfe7adfe4f56bedd968d12470421 2024-11-14T02:44:15,780 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/C of 1d50bec2e36af6979b333798612ed279 into 4f98bfe7adfe4f56bedd968d12470421(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:15,780 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:15,780 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/C, priority=13, startTime=1731552254901; duration=0sec 2024-11-14T02:44:15,780 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:15,780 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:C 2024-11-14T02:44:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:15,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:15,878 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/33404f14b4774e11987b69e7901ff457 2024-11-14T02:44:15,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552315881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:15,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:15,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:15,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:15,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/cd3bfb18d86f4814ad4c8827a01ba4a2 is 50, key is test_row_0/B:col10/1731552253675/Put/seqid=0 2024-11-14T02:44:15,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741986_1162 (size=12151) 2024-11-14T02:44:15,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552315995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:16,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552316205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:16,302 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/cd3bfb18d86f4814ad4c8827a01ba4a2 2024-11-14T02:44:16,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/c5088952f88549968602783ec76e6dfe is 50, key is test_row_0/C:col10/1731552253675/Put/seqid=0 2024-11-14T02:44:16,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741987_1163 (size=12151) 2024-11-14T02:44:16,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552316515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T02:44:16,715 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/c5088952f88549968602783ec76e6dfe 2024-11-14T02:44:16,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/33404f14b4774e11987b69e7901ff457 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/33404f14b4774e11987b69e7901ff457 2024-11-14T02:44:16,725 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/33404f14b4774e11987b69e7901ff457, entries=150, sequenceid=153, filesize=30.4 K 2024-11-14T02:44:16,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/cd3bfb18d86f4814ad4c8827a01ba4a2 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/cd3bfb18d86f4814ad4c8827a01ba4a2 2024-11-14T02:44:16,730 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/cd3bfb18d86f4814ad4c8827a01ba4a2, entries=150, sequenceid=153, filesize=11.9 K 2024-11-14T02:44:16,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/c5088952f88549968602783ec76e6dfe as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/c5088952f88549968602783ec76e6dfe 2024-11-14T02:44:16,738 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/c5088952f88549968602783ec76e6dfe, entries=150, sequenceid=153, filesize=11.9 K 2024-11-14T02:44:16,739 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1d50bec2e36af6979b333798612ed279 in 1699ms, sequenceid=153, compaction requested=false 2024-11-14T02:44:16,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:16,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:16,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-14T02:44:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-14T02:44:16,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-14T02:44:16,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3140 sec 2024-11-14T02:44:16,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.3180 sec 2024-11-14T02:44:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:17,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T02:44:17,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:17,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:17,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:17,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:17,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:17,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:17,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147b1b5cf4ae3e4a2589f53a7118be6b3e_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552257026/Put/seqid=0 2024-11-14T02:44:17,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741988_1164 (size=14794) 2024-11-14T02:44:17,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552317093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:17,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552317205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:17,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552317415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:17,439 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:17,443 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147b1b5cf4ae3e4a2589f53a7118be6b3e_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147b1b5cf4ae3e4a2589f53a7118be6b3e_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:17,444 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/2a3986c8b9d5402f80e30ef470016b8e, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:17,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/2a3986c8b9d5402f80e30ef470016b8e is 175, key is test_row_0/A:col10/1731552257026/Put/seqid=0 2024-11-14T02:44:17,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741989_1165 (size=39749) 2024-11-14T02:44:17,450 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/2a3986c8b9d5402f80e30ef470016b8e 2024-11-14T02:44:17,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/eae20ef9f8ad48029fb15e916d5a193d is 50, key is test_row_0/B:col10/1731552257026/Put/seqid=0 2024-11-14T02:44:17,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741990_1166 (size=12151) 2024-11-14T02:44:17,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552317725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:17,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/eae20ef9f8ad48029fb15e916d5a193d 2024-11-14T02:44:17,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/85893e2a520b4a0a9d308b4981e7b253 is 50, key is test_row_0/C:col10/1731552257026/Put/seqid=0 2024-11-14T02:44:17,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741991_1167 (size=12151) 2024-11-14T02:44:18,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552318235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:18,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/85893e2a520b4a0a9d308b4981e7b253 2024-11-14T02:44:18,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/2a3986c8b9d5402f80e30ef470016b8e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/2a3986c8b9d5402f80e30ef470016b8e 2024-11-14T02:44:18,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/2a3986c8b9d5402f80e30ef470016b8e, entries=200, sequenceid=171, filesize=38.8 K 2024-11-14T02:44:18,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/eae20ef9f8ad48029fb15e916d5a193d as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/eae20ef9f8ad48029fb15e916d5a193d 2024-11-14T02:44:18,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/eae20ef9f8ad48029fb15e916d5a193d, entries=150, sequenceid=171, filesize=11.9 K 2024-11-14T02:44:18,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/85893e2a520b4a0a9d308b4981e7b253 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/85893e2a520b4a0a9d308b4981e7b253 2024-11-14T02:44:18,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/85893e2a520b4a0a9d308b4981e7b253, entries=150, sequenceid=171, filesize=11.9 K 2024-11-14T02:44:18,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1d50bec2e36af6979b333798612ed279 in 1275ms, sequenceid=171, compaction requested=true 2024-11-14T02:44:18,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:18,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:18,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:18,303 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:18,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:18,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:18,303 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:18,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:18,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:18,304 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:18,304 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:18,304 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/A is initiating minor compaction (all files) 2024-11-14T02:44:18,304 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/B is initiating minor compaction (all files) 2024-11-14T02:44:18,304 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/A in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:18,304 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/B in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:18,304 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d06b702d67c34117aae5fcdc6dd560a8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/33404f14b4774e11987b69e7901ff457, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/2a3986c8b9d5402f80e30ef470016b8e] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=99.9 K 2024-11-14T02:44:18,304 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/0695495ac02249e382be884da73f71b3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/cd3bfb18d86f4814ad4c8827a01ba4a2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/eae20ef9f8ad48029fb15e916d5a193d] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=35.9 K 2024-11-14T02:44:18,304 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:18,304 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d06b702d67c34117aae5fcdc6dd560a8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/33404f14b4774e11987b69e7901ff457, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/2a3986c8b9d5402f80e30ef470016b8e] 2024-11-14T02:44:18,304 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 0695495ac02249e382be884da73f71b3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731552252463 2024-11-14T02:44:18,305 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting d06b702d67c34117aae5fcdc6dd560a8, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731552252463 2024-11-14T02:44:18,305 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting cd3bfb18d86f4814ad4c8827a01ba4a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731552253675 2024-11-14T02:44:18,305 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33404f14b4774e11987b69e7901ff457, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731552253675 2024-11-14T02:44:18,305 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting eae20ef9f8ad48029fb15e916d5a193d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731552255875 2024-11-14T02:44:18,305 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a3986c8b9d5402f80e30ef470016b8e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731552255847 2024-11-14T02:44:18,311 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:18,313 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#B#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:18,314 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/5abb77cb173742e78cd04fedc7335303 is 50, key is test_row_0/B:col10/1731552257026/Put/seqid=0 2024-11-14T02:44:18,314 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e65da1afb4d74fed9898328a108f8180_1d50bec2e36af6979b333798612ed279 store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:18,316 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e65da1afb4d74fed9898328a108f8180_1d50bec2e36af6979b333798612ed279, store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:18,316 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e65da1afb4d74fed9898328a108f8180_1d50bec2e36af6979b333798612ed279 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:18,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741992_1168 (size=12561) 2024-11-14T02:44:18,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741993_1169 (size=4469) 2024-11-14T02:44:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T02:44:18,605 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:18,605 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-14T02:44:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T02:44:18,607 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:18,608 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:18,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T02:44:18,727 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#A#compaction#141 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:18,728 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/658077da9cb04c298fcd1c79ddd05199 is 175, key is test_row_0/A:col10/1731552257026/Put/seqid=0 2024-11-14T02:44:18,731 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/5abb77cb173742e78cd04fedc7335303 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/5abb77cb173742e78cd04fedc7335303 2024-11-14T02:44:18,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741994_1170 (size=31515) 2024-11-14T02:44:18,742 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/658077da9cb04c298fcd1c79ddd05199 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/658077da9cb04c298fcd1c79ddd05199 2024-11-14T02:44:18,742 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/B of 1d50bec2e36af6979b333798612ed279 into 5abb77cb173742e78cd04fedc7335303(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:18,742 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:18,742 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/B, priority=13, startTime=1731552258303; duration=0sec 2024-11-14T02:44:18,742 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:18,742 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:B 2024-11-14T02:44:18,742 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:18,745 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:18,746 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/C is initiating minor compaction (all files) 2024-11-14T02:44:18,746 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/C in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:18,746 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/4f98bfe7adfe4f56bedd968d12470421, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/c5088952f88549968602783ec76e6dfe, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/85893e2a520b4a0a9d308b4981e7b253] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=35.9 K 2024-11-14T02:44:18,746 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f98bfe7adfe4f56bedd968d12470421, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731552252463 2024-11-14T02:44:18,747 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting c5088952f88549968602783ec76e6dfe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731552253675 2024-11-14T02:44:18,747 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 85893e2a520b4a0a9d308b4981e7b253, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731552255875 2024-11-14T02:44:18,753 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/A of 1d50bec2e36af6979b333798612ed279 into 658077da9cb04c298fcd1c79ddd05199(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:18,753 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:18,753 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/A, priority=13, startTime=1731552258302; duration=0sec 2024-11-14T02:44:18,755 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:18,755 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:A 2024-11-14T02:44:18,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T02:44:18,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:18,761 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T02:44:18,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:18,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:18,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:18,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:18,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:18,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:18,763 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#C#compaction#142 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:18,763 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/aaee05883d9849c694f697661973904b is 50, key is test_row_0/C:col10/1731552257026/Put/seqid=0 2024-11-14T02:44:18,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741995_1171 (size=12561) 2024-11-14T02:44:18,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fa176405333c45679212d2a44f80adb2_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552257092/Put/seqid=0 2024-11-14T02:44:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:18,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:18,787 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/aaee05883d9849c694f697661973904b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/aaee05883d9849c694f697661973904b 2024-11-14T02:44:18,798 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/C of 1d50bec2e36af6979b333798612ed279 into aaee05883d9849c694f697661973904b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:18,798 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:18,798 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/C, priority=13, startTime=1731552258303; duration=0sec 2024-11-14T02:44:18,799 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:18,799 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:C 2024-11-14T02:44:18,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741996_1172 (size=12304) 2024-11-14T02:44:18,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552318808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:18,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:18,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552318808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:18,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552318810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:18,820 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fa176405333c45679212d2a44f80adb2_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fa176405333c45679212d2a44f80adb2_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:18,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552318811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:18,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/dfd65ad2095b43c0bfbefb1b24067fed, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/dfd65ad2095b43c0bfbefb1b24067fed is 175, key is test_row_0/A:col10/1731552257092/Put/seqid=0 2024-11-14T02:44:18,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741997_1173 (size=31105) 2024-11-14T02:44:18,839 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/dfd65ad2095b43c0bfbefb1b24067fed 2024-11-14T02:44:18,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/e23f9ab1414d42299e8a03526878d696 is 50, key is test_row_0/B:col10/1731552257092/Put/seqid=0 2024-11-14T02:44:18,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741998_1174 (size=12151) 2024-11-14T02:44:18,867 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/e23f9ab1414d42299e8a03526878d696 2024-11-14T02:44:18,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/04e5ae35655144bcb86584d1285cd275 is 50, key is test_row_0/C:col10/1731552257092/Put/seqid=0 2024-11-14T02:44:18,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741999_1175 (size=12151) 2024-11-14T02:44:18,902 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/04e5ae35655144bcb86584d1285cd275 2024-11-14T02:44:18,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/dfd65ad2095b43c0bfbefb1b24067fed as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/dfd65ad2095b43c0bfbefb1b24067fed 2024-11-14T02:44:18,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552318915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:18,917 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/dfd65ad2095b43c0bfbefb1b24067fed, entries=150, sequenceid=190, filesize=30.4 K 2024-11-14T02:44:18,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/e23f9ab1414d42299e8a03526878d696 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e23f9ab1414d42299e8a03526878d696 2024-11-14T02:44:18,922 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e23f9ab1414d42299e8a03526878d696, entries=150, sequenceid=190, filesize=11.9 K 2024-11-14T02:44:18,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/04e5ae35655144bcb86584d1285cd275 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/04e5ae35655144bcb86584d1285cd275 2024-11-14T02:44:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T02:44:18,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552318925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:18,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552318925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:18,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552318925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:18,928 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/04e5ae35655144bcb86584d1285cd275, entries=150, sequenceid=190, filesize=11.9 K 2024-11-14T02:44:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:18,928 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1d50bec2e36af6979b333798612ed279 in 167ms, sequenceid=190, compaction requested=false 2024-11-14T02:44:18,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:18,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:18,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-14T02:44:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-14T02:44:18,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-14T02:44:18,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 322 msec 2024-11-14T02:44:18,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 326 msec 2024-11-14T02:44:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:19,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T02:44:19,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:19,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:19,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:19,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:19,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:19,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:19,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142833ac16e08940e19f8dddb5b2157bb2_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552258809/Put/seqid=0 2024-11-14T02:44:19,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552319154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552319156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742000_1176 (size=14794) 2024-11-14T02:44:19,167 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:19,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552319160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552319160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,172 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142833ac16e08940e19f8dddb5b2157bb2_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142833ac16e08940e19f8dddb5b2157bb2_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:19,176 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d628e365ca7b41ec9487770c8aa22187, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:19,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d628e365ca7b41ec9487770c8aa22187 is 175, key is test_row_0/A:col10/1731552258809/Put/seqid=0 2024-11-14T02:44:19,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742001_1177 (size=39749) 2024-11-14T02:44:19,212 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d628e365ca7b41ec9487770c8aa22187 2024-11-14T02:44:19,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/ec1f47e0cacd4eeaa4253eb2493e0c00 is 50, key is test_row_0/B:col10/1731552258809/Put/seqid=0 2024-11-14T02:44:19,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742002_1178 (size=12151) 2024-11-14T02:44:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T02:44:19,236 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:19,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-14T02:44:19,239 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:19,240 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:19,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:19,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T02:44:19,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552319255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,261 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_0' in 1d50bec2e36af6979b333798612ed279 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552319266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552319266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552319275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552319275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T02:44:19,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T02:44:19,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:19,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:19,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:19,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:19,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:19,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552319475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552319475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552319486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552319486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T02:44:19,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:19,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:19,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:19,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:19,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:19,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T02:44:19,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/ec1f47e0cacd4eeaa4253eb2493e0c00 2024-11-14T02:44:19,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T02:44:19,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:19,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:19,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:19,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:19,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/fa397c3c071a4601a2f472b1c3d358e1 is 50, key is test_row_0/C:col10/1731552258809/Put/seqid=0 2024-11-14T02:44:19,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:19,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742003_1179 (size=12151) 2024-11-14T02:44:19,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552319785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552319785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552319795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552319795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:19,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T02:44:19,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:19,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:19,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:19,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:19,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T02:44:20,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T02:44:20,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:20,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:20,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:20,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/fa397c3c071a4601a2f472b1c3d358e1 2024-11-14T02:44:20,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d628e365ca7b41ec9487770c8aa22187 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d628e365ca7b41ec9487770c8aa22187 2024-11-14T02:44:20,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d628e365ca7b41ec9487770c8aa22187, entries=200, sequenceid=207, filesize=38.8 K 2024-11-14T02:44:20,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/ec1f47e0cacd4eeaa4253eb2493e0c00 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/ec1f47e0cacd4eeaa4253eb2493e0c00 2024-11-14T02:44:20,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/ec1f47e0cacd4eeaa4253eb2493e0c00, entries=150, sequenceid=207, filesize=11.9 K 2024-11-14T02:44:20,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/fa397c3c071a4601a2f472b1c3d358e1 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/fa397c3c071a4601a2f472b1c3d358e1 2024-11-14T02:44:20,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/fa397c3c071a4601a2f472b1c3d358e1, entries=150, sequenceid=207, filesize=11.9 K 2024-11-14T02:44:20,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 1d50bec2e36af6979b333798612ed279 in 999ms, sequenceid=207, compaction requested=true 2024-11-14T02:44:20,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:20,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:20,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:20,125 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:20,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:20,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:20,125 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:20,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:20,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:20,129 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:20,129 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:20,129 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/A is initiating minor compaction (all files) 2024-11-14T02:44:20,129 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/B is initiating minor compaction (all files) 2024-11-14T02:44:20,129 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/A in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,129 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/B in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,129 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/5abb77cb173742e78cd04fedc7335303, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e23f9ab1414d42299e8a03526878d696, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/ec1f47e0cacd4eeaa4253eb2493e0c00] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=36.0 K 2024-11-14T02:44:20,129 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/658077da9cb04c298fcd1c79ddd05199, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/dfd65ad2095b43c0bfbefb1b24067fed, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d628e365ca7b41ec9487770c8aa22187] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=100.0 K 2024-11-14T02:44:20,129 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,129 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/658077da9cb04c298fcd1c79ddd05199, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/dfd65ad2095b43c0bfbefb1b24067fed, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d628e365ca7b41ec9487770c8aa22187] 2024-11-14T02:44:20,130 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 5abb77cb173742e78cd04fedc7335303, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731552255875 2024-11-14T02:44:20,130 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 658077da9cb04c298fcd1c79ddd05199, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731552255875 2024-11-14T02:44:20,130 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting e23f9ab1414d42299e8a03526878d696, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731552257078 2024-11-14T02:44:20,130 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfd65ad2095b43c0bfbefb1b24067fed, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731552257078 2024-11-14T02:44:20,130 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting ec1f47e0cacd4eeaa4253eb2493e0c00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731552258808 2024-11-14T02:44:20,130 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting d628e365ca7b41ec9487770c8aa22187, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731552258806 2024-11-14T02:44:20,135 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:20,137 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#B#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:20,137 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/9bbefb9ed1b243458d033030be3e7899 is 50, key is test_row_0/B:col10/1731552258809/Put/seqid=0 2024-11-14T02:44:20,138 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114974be350281d44caab596ea81027b005_1d50bec2e36af6979b333798612ed279 store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:20,141 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114974be350281d44caab596ea81027b005_1d50bec2e36af6979b333798612ed279, store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:20,141 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114974be350281d44caab596ea81027b005_1d50bec2e36af6979b333798612ed279 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:20,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742004_1180 (size=12663) 2024-11-14T02:44:20,150 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/9bbefb9ed1b243458d033030be3e7899 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/9bbefb9ed1b243458d033030be3e7899 2024-11-14T02:44:20,157 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/B of 1d50bec2e36af6979b333798612ed279 into 9bbefb9ed1b243458d033030be3e7899(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:20,157 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:20,157 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/B, priority=13, startTime=1731552260125; duration=0sec 2024-11-14T02:44:20,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T02:44:20,158 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:20,158 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:B 2024-11-14T02:44:20,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,158 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:20,158 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T02:44:20,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:20,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:20,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:20,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:20,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:20,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:20,159 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:20,159 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/C is initiating minor compaction (all files) 2024-11-14T02:44:20,159 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/C in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,159 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/aaee05883d9849c694f697661973904b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/04e5ae35655144bcb86584d1285cd275, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/fa397c3c071a4601a2f472b1c3d358e1] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=36.0 K 2024-11-14T02:44:20,159 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting aaee05883d9849c694f697661973904b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731552255875 2024-11-14T02:44:20,160 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 04e5ae35655144bcb86584d1285cd275, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731552257078 2024-11-14T02:44:20,161 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting fa397c3c071a4601a2f472b1c3d358e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731552258808 2024-11-14T02:44:20,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742005_1181 (size=4469) 2024-11-14T02:44:20,168 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#A#compaction#150 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:20,169 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d1bc939ebda947ed8ed6accc70961ef6 is 175, key is test_row_0/A:col10/1731552258809/Put/seqid=0 2024-11-14T02:44:20,171 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#C#compaction#151 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:20,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a6284526233a456c933268737ae758f0_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552259154/Put/seqid=0 2024-11-14T02:44:20,172 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/c2dfda81a014455b9aea746ed6576f0e is 50, key is test_row_0/C:col10/1731552258809/Put/seqid=0 2024-11-14T02:44:20,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742006_1182 (size=31617) 2024-11-14T02:44:20,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742007_1183 (size=12663) 2024-11-14T02:44:20,190 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/c2dfda81a014455b9aea746ed6576f0e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/c2dfda81a014455b9aea746ed6576f0e 2024-11-14T02:44:20,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742008_1184 (size=12304) 2024-11-14T02:44:20,196 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/C of 1d50bec2e36af6979b333798612ed279 into c2dfda81a014455b9aea746ed6576f0e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:20,196 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:20,196 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/C, priority=13, startTime=1731552260125; duration=0sec 2024-11-14T02:44:20,196 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:20,197 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:C 2024-11-14T02:44:20,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,219 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a6284526233a456c933268737ae758f0_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a6284526233a456c933268737ae758f0_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:20,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/bab7d0fc3682459c9ac1c6669d78f8cb, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:20,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/bab7d0fc3682459c9ac1c6669d78f8cb is 175, key is test_row_0/A:col10/1731552259154/Put/seqid=0 2024-11-14T02:44:20,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742009_1185 (size=31105) 2024-11-14T02:44:20,227 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/bab7d0fc3682459c9ac1c6669d78f8cb 2024-11-14T02:44:20,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/a219ca1f535040069cda282328f901d7 is 50, key is test_row_0/B:col10/1731552259154/Put/seqid=0 2024-11-14T02:44:20,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742010_1186 (size=12151) 2024-11-14T02:44:20,260 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/a219ca1f535040069cda282328f901d7 2024-11-14T02:44:20,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/777570e0df7948d8af0ce4dffd6b2fe1 is 50, key is test_row_0/C:col10/1731552259154/Put/seqid=0 2024-11-14T02:44:20,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742011_1187 (size=12151) 2024-11-14T02:44:20,277 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/777570e0df7948d8af0ce4dffd6b2fe1 2024-11-14T02:44:20,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/bab7d0fc3682459c9ac1c6669d78f8cb as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/bab7d0fc3682459c9ac1c6669d78f8cb 2024-11-14T02:44:20,288 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/bab7d0fc3682459c9ac1c6669d78f8cb, entries=150, sequenceid=225, filesize=30.4 K 2024-11-14T02:44:20,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/a219ca1f535040069cda282328f901d7 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/a219ca1f535040069cda282328f901d7 2024-11-14T02:44:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,295 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/a219ca1f535040069cda282328f901d7, entries=150, sequenceid=225, filesize=11.9 K 2024-11-14T02:44:20,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/777570e0df7948d8af0ce4dffd6b2fe1 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/777570e0df7948d8af0ce4dffd6b2fe1 2024-11-14T02:44:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,302 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/777570e0df7948d8af0ce4dffd6b2fe1, entries=150, sequenceid=225, filesize=11.9 K 2024-11-14T02:44:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,303 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 1d50bec2e36af6979b333798612ed279 in 145ms, sequenceid=225, compaction requested=false 2024-11-14T02:44:20,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:20,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-14T02:44:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-14T02:44:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,307 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-14T02:44:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0650 sec 2024-11-14T02:44:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.0720 sec 2024-11-14T02:44:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:20,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:44:20,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:20,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:20,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:20,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:20,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T02:44:20,375 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-14T02:44:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,379 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T02:44:20,379 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:20,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111430df3170ee0f44ddbcb26e38ed0cd5fc_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552260370/Put/seqid=0 2024-11-14T02:44:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742012_1188 (size=17284) 2024-11-14T02:44:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,399 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,405 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111430df3170ee0f44ddbcb26e38ed0cd5fc_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111430df3170ee0f44ddbcb26e38ed0cd5fc_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,406 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/13df687fd22446d2901c10e0fb7ce7d7, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/13df687fd22446d2901c10e0fb7ce7d7 is 175, key is test_row_0/A:col10/1731552260370/Put/seqid=0 2024-11-14T02:44:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:20,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742013_1189 (size=48389) 2024-11-14T02:44:20,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731552260402, max timestamp=1731552260420Num uniques 859; 2024-11-14T02:44:20,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731552260402, max timestamp=1731552260420Num uniques 859; 2024-11-14T02:44:20,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T02:44:20,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T02:44:20,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731552260414, max timestamp=1731552260420Num uniques 859; 2024-11-14T02:44:20,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:44:20,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552320446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:20,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552320475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:20,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552320475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:20,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552320475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T02:44:20,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:44:20,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:20,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:20,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:20,588 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/d1bc939ebda947ed8ed6accc70961ef6 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d1bc939ebda947ed8ed6accc70961ef6 2024-11-14T02:44:20,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552320585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552320587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:20,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552320587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:20,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552320596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:20,597 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/A of 1d50bec2e36af6979b333798612ed279 into d1bc939ebda947ed8ed6accc70961ef6(size=30.9 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,597 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,597 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/A, priority=13, startTime=1731552260125; duration=0sec 2024-11-14T02:44:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:20,597 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:20,597 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:A 2024-11-14T02:44:20,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:44:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:20,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T02:44:20,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:20,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552320796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:20,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552320805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:20,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552320805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:20,812 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/13df687fd22446d2901c10e0fb7ce7d7 2024-11-14T02:44:20,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552320810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:20,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/9356adbf8db842c8a7e4d67be1447da4 is 50, key is test_row_0/B:col10/1731552260370/Put/seqid=0 2024-11-14T02:44:20,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742014_1190 (size=12151) 2024-11-14T02:44:20,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/9356adbf8db842c8a7e4d67be1447da4 2024-11-14T02:44:20,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:44:20,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:20,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:20,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:20,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/36b750b797574fbfbaa53d7bfbf7df34 is 50, key is test_row_0/C:col10/1731552260370/Put/seqid=0 2024-11-14T02:44:20,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742015_1191 (size=12151) 2024-11-14T02:44:20,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:44:20,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:20,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:20,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:20,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T02:44:21,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552321105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552321115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552321115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:21,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552321125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:21,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:21,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:44:21,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:21,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/36b750b797574fbfbaa53d7bfbf7df34 2024-11-14T02:44:21,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/13df687fd22446d2901c10e0fb7ce7d7 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/13df687fd22446d2901c10e0fb7ce7d7 2024-11-14T02:44:21,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/13df687fd22446d2901c10e0fb7ce7d7, entries=250, sequenceid=236, filesize=47.3 K 2024-11-14T02:44:21,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/9356adbf8db842c8a7e4d67be1447da4 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/9356adbf8db842c8a7e4d67be1447da4 2024-11-14T02:44:21,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/9356adbf8db842c8a7e4d67be1447da4, entries=150, sequenceid=236, filesize=11.9 K 2024-11-14T02:44:21,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/36b750b797574fbfbaa53d7bfbf7df34 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/36b750b797574fbfbaa53d7bfbf7df34 2024-11-14T02:44:21,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/36b750b797574fbfbaa53d7bfbf7df34, entries=150, sequenceid=236, filesize=11.9 K 2024-11-14T02:44:21,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=145.36 KB/148850 for 1d50bec2e36af6979b333798612ed279 in 904ms, sequenceid=236, compaction requested=true 2024-11-14T02:44:21,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:21,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:21,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:21,275 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:21,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:21,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:21,275 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:21,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:21,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:21,276 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111111 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:21,276 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:21,276 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/A is initiating minor compaction (all files) 2024-11-14T02:44:21,276 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/B is initiating minor compaction (all files) 2024-11-14T02:44:21,276 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/A in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,276 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/B in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,276 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d1bc939ebda947ed8ed6accc70961ef6, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/bab7d0fc3682459c9ac1c6669d78f8cb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/13df687fd22446d2901c10e0fb7ce7d7] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=108.5 K 2024-11-14T02:44:21,276 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/9bbefb9ed1b243458d033030be3e7899, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/a219ca1f535040069cda282328f901d7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/9356adbf8db842c8a7e4d67be1447da4] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=36.1 K 2024-11-14T02:44:21,276 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,276 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d1bc939ebda947ed8ed6accc70961ef6, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/bab7d0fc3682459c9ac1c6669d78f8cb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/13df687fd22446d2901c10e0fb7ce7d7] 2024-11-14T02:44:21,277 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1bc939ebda947ed8ed6accc70961ef6, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731552258808 2024-11-14T02:44:21,277 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bbefb9ed1b243458d033030be3e7899, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731552258808 2024-11-14T02:44:21,277 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting bab7d0fc3682459c9ac1c6669d78f8cb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731552259154 2024-11-14T02:44:21,277 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting a219ca1f535040069cda282328f901d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731552259154 2024-11-14T02:44:21,277 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13df687fd22446d2901c10e0fb7ce7d7, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731552260348 2024-11-14T02:44:21,277 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 9356adbf8db842c8a7e4d67be1447da4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731552260367 2024-11-14T02:44:21,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:21,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=152.07 KB heapSize=425.61 KB 2024-11-14T02:44:21,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:21,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:21,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:21,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:21,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:21,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:21,286 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:21,286 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#B#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:21,287 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/3c599224e7dd4c7fba949626c711b04b is 50, key is test_row_0/B:col10/1731552260370/Put/seqid=0 2024-11-14T02:44:21,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d09e32e1df5346229fd2ae09eb319c93_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552261278/Put/seqid=0 2024-11-14T02:44:21,288 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411145e6e0f75514c4fb7a567c6adaa59dbfe_1d50bec2e36af6979b333798612ed279 store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:21,290 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411145e6e0f75514c4fb7a567c6adaa59dbfe_1d50bec2e36af6979b333798612ed279, store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:21,290 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145e6e0f75514c4fb7a567c6adaa59dbfe_1d50bec2e36af6979b333798612ed279 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:21,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:44:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742016_1192 (size=12765) 2024-11-14T02:44:21,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742017_1193 (size=14994) 2024-11-14T02:44:21,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552321301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:21,313 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/3c599224e7dd4c7fba949626c711b04b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/3c599224e7dd4c7fba949626c711b04b 2024-11-14T02:44:21,318 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/B of 1d50bec2e36af6979b333798612ed279 into 3c599224e7dd4c7fba949626c711b04b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:21,318 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:21,318 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/B, priority=13, startTime=1731552261275; duration=0sec 2024-11-14T02:44:21,318 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:21,318 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:B 2024-11-14T02:44:21,318 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:21,319 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:21,319 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 1d50bec2e36af6979b333798612ed279/C is initiating minor compaction (all files) 2024-11-14T02:44:21,319 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d50bec2e36af6979b333798612ed279/C in TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,319 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/c2dfda81a014455b9aea746ed6576f0e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/777570e0df7948d8af0ce4dffd6b2fe1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/36b750b797574fbfbaa53d7bfbf7df34] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp, totalSize=36.1 K 2024-11-14T02:44:21,320 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting c2dfda81a014455b9aea746ed6576f0e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731552258808 2024-11-14T02:44:21,320 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 777570e0df7948d8af0ce4dffd6b2fe1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731552259154 2024-11-14T02:44:21,320 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b750b797574fbfbaa53d7bfbf7df34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731552260367 2024-11-14T02:44:21,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742018_1194 (size=4469) 2024-11-14T02:44:21,327 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#C#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:21,328 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/7f3da944763d4f49ace362184ca56963 is 50, key is test_row_0/C:col10/1731552260370/Put/seqid=0 2024-11-14T02:44:21,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742019_1195 (size=12765) 2024-11-14T02:44:21,366 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/7f3da944763d4f49ace362184ca56963 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/7f3da944763d4f49ace362184ca56963 2024-11-14T02:44:21,372 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/C of 1d50bec2e36af6979b333798612ed279 into 7f3da944763d4f49ace362184ca56963(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:21,373 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:21,373 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/C, priority=13, startTime=1731552261275; duration=0sec 2024-11-14T02:44:21,373 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:21,373 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:C 2024-11-14T02:44:21,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552321415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:21,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:21,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:44:21,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:21,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T02:44:21,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:44:21,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:21,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552321626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:21,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552321626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552321626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552321626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:21,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552321645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:21,707 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:21,712 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d09e32e1df5346229fd2ae09eb319c93_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d09e32e1df5346229fd2ae09eb319c93_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:21,713 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/e8f0ce6dbcd443189010d4d1d431d90d, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:21,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/e8f0ce6dbcd443189010d4d1d431d90d is 175, key is test_row_0/A:col10/1731552261278/Put/seqid=0 2024-11-14T02:44:21,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742020_1196 (size=39949) 2024-11-14T02:44:21,723 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d50bec2e36af6979b333798612ed279#A#compaction#160 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:44:21,724 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/fec8c6d7c424445e94246e9a7f6e6727 is 175, key is test_row_0/A:col10/1731552260370/Put/seqid=0 2024-11-14T02:44:21,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742021_1197 (size=31719) 2024-11-14T02:44:21,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:44:21,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:21,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:44:21,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:21,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:21,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:21,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552321935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:22,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:44:22,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:22,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:22,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:22,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:22,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:22,118 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/e8f0ce6dbcd443189010d4d1d431d90d 2024-11-14T02:44:22,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/1323d3ebfca543ae8b1428500376a198 is 50, key is test_row_0/B:col10/1731552261278/Put/seqid=0 2024-11-14T02:44:22,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742022_1198 (size=12301) 2024-11-14T02:44:22,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/1323d3ebfca543ae8b1428500376a198 2024-11-14T02:44:22,133 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/fec8c6d7c424445e94246e9a7f6e6727 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/fec8c6d7c424445e94246e9a7f6e6727 2024-11-14T02:44:22,138 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d50bec2e36af6979b333798612ed279/A of 1d50bec2e36af6979b333798612ed279 into fec8c6d7c424445e94246e9a7f6e6727(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:22,138 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:22,138 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., storeName=1d50bec2e36af6979b333798612ed279/A, priority=13, startTime=1731552261275; duration=0sec 2024-11-14T02:44:22,139 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:22,139 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:A 2024-11-14T02:44:22,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/6c1bdd053ab5424bad740101867148df is 50, key is test_row_0/C:col10/1731552261278/Put/seqid=0 2024-11-14T02:44:22,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742023_1199 (size=12301) 2024-11-14T02:44:22,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/6c1bdd053ab5424bad740101867148df 2024-11-14T02:44:22,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/e8f0ce6dbcd443189010d4d1d431d90d as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/e8f0ce6dbcd443189010d4d1d431d90d 2024-11-14T02:44:22,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/e8f0ce6dbcd443189010d4d1d431d90d, entries=200, sequenceid=274, filesize=39.0 K 2024-11-14T02:44:22,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/1323d3ebfca543ae8b1428500376a198 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/1323d3ebfca543ae8b1428500376a198 2024-11-14T02:44:22,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/1323d3ebfca543ae8b1428500376a198, entries=150, sequenceid=274, filesize=12.0 K 2024-11-14T02:44:22,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/6c1bdd053ab5424bad740101867148df as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/6c1bdd053ab5424bad740101867148df 2024-11-14T02:44:22,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/6c1bdd053ab5424bad740101867148df, entries=150, sequenceid=274, filesize=12.0 K 2024-11-14T02:44:22,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~152.07 KB/155720, heapSize ~425.56 KB/435776, currentSize=33.54 KB/34350 for 1d50bec2e36af6979b333798612ed279 in 895ms, sequenceid=274, compaction requested=false 2024-11-14T02:44:22,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:22,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T02:44:22,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:22,216 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T02:44:22,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:22,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:22,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:22,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:22,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:22,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:22,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e61e97c7366f4e5380379c3d7c6f1d6f_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552261282/Put/seqid=0 2024-11-14T02:44:22,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742024_1200 (size=12454) 2024-11-14T02:44:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:22,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:22,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552262495, max timestamp=1731552262513Num uniques 859; 2024-11-14T02:44:22,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552262495, max timestamp=1731552262513Num uniques 859; 2024-11-14T02:44:22,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:22,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:22,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552262495, max timestamp=1731552262513Num uniques 859; 2024-11-14T02:44:22,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T02:44:22,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731552262569, max timestamp=1731552262596Num uniques 1068; 2024-11-14T02:44:22,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-14T02:44:22,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731552262569, max timestamp=1731552262596Num uniques 1068; 2024-11-14T02:44:22,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731552262569, max timestamp=1731552262596Num uniques 1068; 2024-11-14T02:44:22,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-14T02:44:22,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-14T02:44:22,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:22,637 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e61e97c7366f4e5380379c3d7c6f1d6f_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e61e97c7366f4e5380379c3d7c6f1d6f_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:22,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/01fc9030cbee42e6bf1cb298d176c986, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:22,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/01fc9030cbee42e6bf1cb298d176c986 is 175, key is test_row_0/A:col10/1731552261282/Put/seqid=0 2024-11-14T02:44:22,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742025_1201 (size=31255) 2024-11-14T02:44:22,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731552262636, max timestamp=1731552262650Num uniques 1077; 2024-11-14T02:44:22,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731552262636, max timestamp=1731552262650Num uniques 1077; 2024-11-14T02:44:22,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-14T02:44:22,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-14T02:44:22,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731552262636, max timestamp=1731552262650Num uniques 1077; 2024-11-14T02:44:22,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-14T02:44:22,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552322700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:22,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552322700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:22,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552322701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552322702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:22,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552322707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:22,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552322815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552322815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552322816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552322816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:22,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552322825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552323025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:23,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552323026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552323026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:23,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552323026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552323035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,043 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/01fc9030cbee42e6bf1cb298d176c986 2024-11-14T02:44:23,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/fbbdca010c0d4a0ca9b7e975ff64073d is 50, key is test_row_0/B:col10/1731552261282/Put/seqid=0 2024-11-14T02:44:23,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742026_1202 (size=12301) 2024-11-14T02:44:23,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552323335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:23,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552323335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552323335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:23,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552323335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552323345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,455 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/fbbdca010c0d4a0ca9b7e975ff64073d 2024-11-14T02:44:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/ad7abcf8f7354e0aa57317a7855dc114 is 50, key is test_row_0/C:col10/1731552261282/Put/seqid=0 2024-11-14T02:44:23,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742027_1203 (size=12301) 2024-11-14T02:44:23,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552323845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:23,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552323845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552323855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:23,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552323856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552323856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:23,866 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/ad7abcf8f7354e0aa57317a7855dc114 2024-11-14T02:44:23,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/01fc9030cbee42e6bf1cb298d176c986 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/01fc9030cbee42e6bf1cb298d176c986 2024-11-14T02:44:23,874 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/01fc9030cbee42e6bf1cb298d176c986, entries=150, sequenceid=285, filesize=30.5 K 2024-11-14T02:44:23,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/fbbdca010c0d4a0ca9b7e975ff64073d as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/fbbdca010c0d4a0ca9b7e975ff64073d 2024-11-14T02:44:23,879 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/fbbdca010c0d4a0ca9b7e975ff64073d, entries=150, sequenceid=285, filesize=12.0 K 2024-11-14T02:44:23,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/ad7abcf8f7354e0aa57317a7855dc114 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/ad7abcf8f7354e0aa57317a7855dc114 2024-11-14T02:44:23,884 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/ad7abcf8f7354e0aa57317a7855dc114, entries=150, sequenceid=285, filesize=12.0 K 2024-11-14T02:44:23,885 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 1d50bec2e36af6979b333798612ed279 in 1669ms, sequenceid=285, compaction requested=true 2024-11-14T02:44:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-14T02:44:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-14T02:44:23,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-14T02:44:23,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5070 sec 2024-11-14T02:44:23,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 3.5110 sec 2024-11-14T02:44:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T02:44:24,555 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:24,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-14T02:44:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T02:44:24,557 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:24,558 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:24,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T02:44:24,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T02:44:24,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:24,710 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=154.31 KB heapSize=429 KB 2024-11-14T02:44:24,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:24,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:24,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:24,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:24,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:24,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:24,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b607fbe446fc445da2c8dc47f98620a4_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552262700/Put/seqid=0 2024-11-14T02:44:24,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742028_1204 (size=12454) 2024-11-14T02:44:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:24,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:24,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552324866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:24,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552324868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:24,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552324869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552324870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T02:44:24,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552324875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:24,876 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_0' in 1d50bec2e36af6979b333798612ed279 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:24,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552324975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:24,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552324975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:24,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552324975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:24,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:24,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:24,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552324975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:24,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:24,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:24,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T02:44:25,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552325185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:25,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552325185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:25,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552325185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:25,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:25,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:25,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552325185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:25,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:25,195 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T02:44:25,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:25,229 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b607fbe446fc445da2c8dc47f98620a4_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b607fbe446fc445da2c8dc47f98620a4_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:25,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/675dcb74204249c1950e7c45d532685f, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:25,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/675dcb74204249c1950e7c45d532685f is 175, key is test_row_0/A:col10/1731552262700/Put/seqid=0 2024-11-14T02:44:25,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742029_1205 (size=31255) 2024-11-14T02:44:25,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552325495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:25,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:25,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552325495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:25,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:25,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:25,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552325495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:25,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:25,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:25,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552325495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:25,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:25,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:25,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:25,635 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/675dcb74204249c1950e7c45d532685f 2024-11-14T02:44:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/106a91ddd8b645f0a9b3022c80eeb718 is 50, key is test_row_0/B:col10/1731552262700/Put/seqid=0 2024-11-14T02:44:25,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742030_1206 (size=12301) 2024-11-14T02:44:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T02:44:26,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552326005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:26,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552326005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:26,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552326005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552326005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:26,046 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/106a91ddd8b645f0a9b3022c80eeb718 2024-11-14T02:44:26,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/7c895a759a81401ab64b718523be994d is 50, key is test_row_0/C:col10/1731552262700/Put/seqid=0 2024-11-14T02:44:26,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742031_1207 (size=12301) 2024-11-14T02:44:26,460 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/7c895a759a81401ab64b718523be994d 2024-11-14T02:44:26,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/675dcb74204249c1950e7c45d532685f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/675dcb74204249c1950e7c45d532685f 2024-11-14T02:44:26,468 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/675dcb74204249c1950e7c45d532685f, entries=150, sequenceid=356, filesize=30.5 K 2024-11-14T02:44:26,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/106a91ddd8b645f0a9b3022c80eeb718 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/106a91ddd8b645f0a9b3022c80eeb718 2024-11-14T02:44:26,474 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/106a91ddd8b645f0a9b3022c80eeb718, entries=150, sequenceid=356, filesize=12.0 K 2024-11-14T02:44:26,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/7c895a759a81401ab64b718523be994d as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/7c895a759a81401ab64b718523be994d 2024-11-14T02:44:26,479 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/7c895a759a81401ab64b718523be994d, entries=150, sequenceid=356, filesize=12.0 K 2024-11-14T02:44:26,480 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~428.95 KB/439248, currentSize=33.54 KB/34350 for 1d50bec2e36af6979b333798612ed279 in 1770ms, sequenceid=356, compaction requested=true 2024-11-14T02:44:26,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:26,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:26,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-14T02:44:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-14T02:44:26,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-14T02:44:26,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9240 sec 2024-11-14T02:44:26,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.9280 sec 2024-11-14T02:44:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T02:44:26,706 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:26,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-14T02:44:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T02:44:26,707 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:26,708 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:26,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T02:44:26,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T02:44:26,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:26,860 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T02:44:26,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:26,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:26,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:26,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:26,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:26,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:26,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114db7bbad4d767476cbc143040a5727341_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552264865/Put/seqid=0 2024-11-14T02:44:26,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742032_1208 (size=12454) 2024-11-14T02:44:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:26,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. as already flushing 2024-11-14T02:44:26,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552266929, max timestamp=1731552266962Num uniques 859; 2024-11-14T02:44:26,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:44:26,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552266929, max timestamp=1731552266962Num uniques 859; 2024-11-14T02:44:26,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:44:26,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552266929, max timestamp=1731552266962Num uniques 859; 2024-11-14T02:44:26,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:44:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T02:44:27,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731552266986, max timestamp=1731552267027Num uniques 1018; 2024-11-14T02:44:27,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-14T02:44:27,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731552266986, max timestamp=1731552267027Num uniques 1018; 2024-11-14T02:44:27,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731552266986, max timestamp=1731552267027Num uniques 1018; 2024-11-14T02:44:27,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-14T02:44:27,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-14T02:44:27,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552327046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552327046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552327047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552327048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552327062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552327166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552327166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552327166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552327166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552327166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:27,275 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114db7bbad4d767476cbc143040a5727341_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114db7bbad4d767476cbc143040a5727341_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:27,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/181c26e9ac8b4aafa7dd2f3af3817afd, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:27,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/181c26e9ac8b4aafa7dd2f3af3817afd is 175, key is test_row_0/A:col10/1731552264865/Put/seqid=0 2024-11-14T02:44:27,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742033_1209 (size=31255) 2024-11-14T02:44:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T02:44:27,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552327375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552327375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552327375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552327375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552327385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,680 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/181c26e9ac8b4aafa7dd2f3af3817afd 2024-11-14T02:44:27,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552327685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552327686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552327686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552327686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/6c7dd43e29374c66911d0fa072920a3e is 50, key is test_row_0/B:col10/1731552264865/Put/seqid=0 2024-11-14T02:44:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742034_1210 (size=12301) 2024-11-14T02:44:27,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552327695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:27,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:27,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T02:44:28,092 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/6c7dd43e29374c66911d0fa072920a3e 2024-11-14T02:44:28,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/8272cf8548c144089b49502cbb7f6dc2 is 50, key is test_row_0/C:col10/1731552264865/Put/seqid=0 2024-11-14T02:44:28,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742035_1211 (size=12301) 2024-11-14T02:44:28,102 INFO [Thread-621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-621. 2024-11-14T02:44:28,103 DEBUG [Thread-621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:28,103 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:28,103 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:28,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:28,104 INFO [Thread-619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-619. 2024-11-14T02:44:28,104 DEBUG [Thread-619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:28,104 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:28,104 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:28,104 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:28,106 INFO [Thread-623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-623. 2024-11-14T02:44:28,106 DEBUG [Thread-623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:28,106 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:28,106 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:28,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:28,118 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-11-14T02:44:28,118 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:28,118 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:28,118 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:28,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:28,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:28,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:28,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33744 deadline: 1731552328196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33712 deadline: 1731552328195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33704 deadline: 1731552328196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:28,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731552328196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:28,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33688 deadline: 1731552328216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:28,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:44:28,504 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/8272cf8548c144089b49502cbb7f6dc2 2024-11-14T02:44:28,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/181c26e9ac8b4aafa7dd2f3af3817afd as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/181c26e9ac8b4aafa7dd2f3af3817afd 2024-11-14T02:44:28,520 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/181c26e9ac8b4aafa7dd2f3af3817afd, entries=150, sequenceid=364, filesize=30.5 K 2024-11-14T02:44:28,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/6c7dd43e29374c66911d0fa072920a3e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/6c7dd43e29374c66911d0fa072920a3e 2024-11-14T02:44:28,526 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/6c7dd43e29374c66911d0fa072920a3e, entries=150, sequenceid=364, filesize=12.0 K 2024-11-14T02:44:28,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/8272cf8548c144089b49502cbb7f6dc2 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/8272cf8548c144089b49502cbb7f6dc2 2024-11-14T02:44:28,531 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/8272cf8548c144089b49502cbb7f6dc2, entries=150, sequenceid=364, filesize=12.0 K 2024-11-14T02:44:28,532 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 1d50bec2e36af6979b333798612ed279 in 1672ms, sequenceid=364, compaction requested=true 2024-11-14T02:44:28,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:28,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:28,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-14T02:44:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-14T02:44:28,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-14T02:44:28,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8250 sec 2024-11-14T02:44:28,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.8300 sec 2024-11-14T02:44:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T02:44:28,856 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:29,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=167.72 KB heapSize=475.08 KB 2024-11-14T02:44:29,212 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-11-14T02:44:29,212 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-11-14T02:44:29,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:29,212 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:29,212 INFO [Thread-608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-608. 2024-11-14T02:44:29,212 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:29,212 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:29,212 DEBUG [Thread-608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:29,212 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:29,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T02:44:29,212 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:29,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:29,212 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:29,212 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:29,212 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:29,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T02:44:29,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:29,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:29,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T02:44:29,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:29,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:29,217 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-11-14T02:44:29,217 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:29,217 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:29,217 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:29,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:29,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d8a084286ffd49e3aa06a26504327e28_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552269207/Put/seqid=0 2024-11-14T02:44:29,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742036_1212 (size=12454) 2024-11-14T02:44:29,226 INFO [Thread-616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-616. 2024-11-14T02:44:29,226 DEBUG [Thread-616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:29,226 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:29,226 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:29,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:29,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T02:44:29,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-14T02:44:29,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-14T02:44:29,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-11-14T02:44:29,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-14T02:44:29,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-14T02:44:29,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T02:44:29,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3167 2024-11-14T02:44:29,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3004 2024-11-14T02:44:29,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T02:44:29,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1313 2024-11-14T02:44:29,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3936 rows 2024-11-14T02:44:29,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1313 2024-11-14T02:44:29,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3933 rows 2024-11-14T02:44:29,226 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:44:29,227 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:44:29,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:29,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:29,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:29,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T02:44:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T02:44:29,230 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552269230"}]},"ts":"1731552269230"} 2024-11-14T02:44:29,231 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T02:44:29,231 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T02:44:29,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:44:29,233 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d50bec2e36af6979b333798612ed279, UNASSIGN}] 2024-11-14T02:44:29,234 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d50bec2e36af6979b333798612ed279, UNASSIGN 2024-11-14T02:44:29,235 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=1d50bec2e36af6979b333798612ed279, regionState=CLOSING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:29,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d50bec2e36af6979b333798612ed279, UNASSIGN because future has completed 2024-11-14T02:44:29,236 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:44:29,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:44:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T02:44:29,389 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:29,389 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:44:29,389 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing 1d50bec2e36af6979b333798612ed279, disabling compactions & flushes 2024-11-14T02:44:29,390 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:29,529 DEBUG [regionserver/e5dc8b5cf5e9:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.15384615384615385, tune throughput to 57.69 MB/second 2024-11-14T02:44:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T02:44:29,625 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:29,633 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d8a084286ffd49e3aa06a26504327e28_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d8a084286ffd49e3aa06a26504327e28_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:29,635 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/76cbe1ff29e745b7b154de0fbcd98d71, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:29,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/76cbe1ff29e745b7b154de0fbcd98d71 is 175, key is test_row_0/A:col10/1731552269207/Put/seqid=0 2024-11-14T02:44:29,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742037_1213 (size=31255) 2024-11-14T02:44:29,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T02:44:30,041 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/76cbe1ff29e745b7b154de0fbcd98d71 2024-11-14T02:44:30,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/0959dbb0e8084ffea6d306f78700649c is 50, key is test_row_0/B:col10/1731552269207/Put/seqid=0 2024-11-14T02:44:30,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742038_1214 (size=12301) 2024-11-14T02:44:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T02:44:30,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/0959dbb0e8084ffea6d306f78700649c 2024-11-14T02:44:30,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/963fe027ce2447b6b0430753baf9b4f7 is 50, key is test_row_0/C:col10/1731552269207/Put/seqid=0 2024-11-14T02:44:30,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742039_1215 (size=12301) 2024-11-14T02:44:30,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/963fe027ce2447b6b0430753baf9b4f7 2024-11-14T02:44:30,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/76cbe1ff29e745b7b154de0fbcd98d71 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/76cbe1ff29e745b7b154de0fbcd98d71 2024-11-14T02:44:30,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/76cbe1ff29e745b7b154de0fbcd98d71, entries=150, sequenceid=421, filesize=30.5 K 2024-11-14T02:44:30,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/0959dbb0e8084ffea6d306f78700649c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/0959dbb0e8084ffea6d306f78700649c 2024-11-14T02:44:30,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/0959dbb0e8084ffea6d306f78700649c, entries=150, sequenceid=421, filesize=12.0 K 2024-11-14T02:44:30,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/963fe027ce2447b6b0430753baf9b4f7 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/963fe027ce2447b6b0430753baf9b4f7 2024-11-14T02:44:30,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/963fe027ce2447b6b0430753baf9b4f7, entries=150, sequenceid=421, filesize=12.0 K 2024-11-14T02:44:30,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~475.03 KB/486432, currentSize=13.42 KB/13740 for 1d50bec2e36af6979b333798612ed279 in 1773ms, sequenceid=421, compaction requested=true 2024-11-14T02:44:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d50bec2e36af6979b333798612ed279: 2024-11-14T02:44:30,984 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:30,984 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:30,984 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. after waiting 0 ms 2024-11-14T02:44:30,984 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. because compaction request was cancelled 2024-11-14T02:44:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:30,984 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d50bec2e36af6979b333798612ed279:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:30,984 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:A 2024-11-14T02:44:30,984 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. because compaction request was cancelled 2024-11-14T02:44:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:30,984 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:B 2024-11-14T02:44:30,984 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. because compaction request was cancelled 2024-11-14T02:44:30,984 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d50bec2e36af6979b333798612ed279:C 2024-11-14T02:44:30,984 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing 1d50bec2e36af6979b333798612ed279 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-14T02:44:30,985 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=A 2024-11-14T02:44:30,985 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:30,985 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=B 2024-11-14T02:44:30,985 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:30,985 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d50bec2e36af6979b333798612ed279, store=C 2024-11-14T02:44:30,985 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:30,990 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b780b1929d504b189a009a977e7518db_1d50bec2e36af6979b333798612ed279 is 50, key is test_row_0/A:col10/1731552269216/Put/seqid=0 2024-11-14T02:44:30,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742040_1216 (size=9914) 2024-11-14T02:44:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T02:44:31,395 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:31,402 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b780b1929d504b189a009a977e7518db_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b780b1929d504b189a009a977e7518db_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:31,404 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/e7b7d6941d064a2f8a59c34b5b00b501, store: [table=TestAcidGuarantees family=A region=1d50bec2e36af6979b333798612ed279] 2024-11-14T02:44:31,405 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/e7b7d6941d064a2f8a59c34b5b00b501 is 175, key is test_row_0/A:col10/1731552269216/Put/seqid=0 2024-11-14T02:44:31,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742041_1217 (size=22561) 2024-11-14T02:44:31,811 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=426, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/e7b7d6941d064a2f8a59c34b5b00b501 2024-11-14T02:44:31,824 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/2c8fa969a0884f658bf4db4a721b7ae5 is 50, key is test_row_0/B:col10/1731552269216/Put/seqid=0 2024-11-14T02:44:31,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742042_1218 (size=9857) 2024-11-14T02:44:32,229 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/2c8fa969a0884f658bf4db4a721b7ae5 2024-11-14T02:44:32,238 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/a907951a5fb7484791e4c00b41cbccba is 50, key is test_row_0/C:col10/1731552269216/Put/seqid=0 2024-11-14T02:44:32,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742043_1219 (size=9857) 2024-11-14T02:44:32,643 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/a907951a5fb7484791e4c00b41cbccba 2024-11-14T02:44:32,652 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/A/e7b7d6941d064a2f8a59c34b5b00b501 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/e7b7d6941d064a2f8a59c34b5b00b501 2024-11-14T02:44:32,659 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/e7b7d6941d064a2f8a59c34b5b00b501, entries=100, sequenceid=426, filesize=22.0 K 2024-11-14T02:44:32,660 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/B/2c8fa969a0884f658bf4db4a721b7ae5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/2c8fa969a0884f658bf4db4a721b7ae5 2024-11-14T02:44:32,666 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/2c8fa969a0884f658bf4db4a721b7ae5, entries=100, sequenceid=426, filesize=9.6 K 2024-11-14T02:44:32,667 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/.tmp/C/a907951a5fb7484791e4c00b41cbccba as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/a907951a5fb7484791e4c00b41cbccba 2024-11-14T02:44:32,672 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/a907951a5fb7484791e4c00b41cbccba, entries=100, sequenceid=426, filesize=9.6 K 2024-11-14T02:44:32,673 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 1d50bec2e36af6979b333798612ed279 in 1688ms, sequenceid=426, compaction requested=true 2024-11-14T02:44:32,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d309599af204450f9c5475482328664f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/696c8501f31c41638b164b282ddaceff, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d9c096da6cfb4ab6ae045de6c8357284, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/b9c12bcc27f94717a4f0aa736b4cb7c6, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/1185202aa0e348649e054ed991f7d727, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/c46ddbbdfc6d440a8bbf5f3fcd9e51cd, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d436d32a42af49429b430297f25f2cf0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/1a5ac89bd6584dde9b661b02bb3e47d0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d06b702d67c34117aae5fcdc6dd560a8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/ac75e8708bd04df1bb4f5518640be40b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/33404f14b4774e11987b69e7901ff457, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/2a3986c8b9d5402f80e30ef470016b8e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/658077da9cb04c298fcd1c79ddd05199, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/dfd65ad2095b43c0bfbefb1b24067fed, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d628e365ca7b41ec9487770c8aa22187, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d1bc939ebda947ed8ed6accc70961ef6, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/bab7d0fc3682459c9ac1c6669d78f8cb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/13df687fd22446d2901c10e0fb7ce7d7] to archive 2024-11-14T02:44:32,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:44:32,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d309599af204450f9c5475482328664f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d309599af204450f9c5475482328664f 2024-11-14T02:44:32,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/696c8501f31c41638b164b282ddaceff to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/696c8501f31c41638b164b282ddaceff 2024-11-14T02:44:32,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d9c096da6cfb4ab6ae045de6c8357284 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d9c096da6cfb4ab6ae045de6c8357284 2024-11-14T02:44:32,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/b9c12bcc27f94717a4f0aa736b4cb7c6 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/b9c12bcc27f94717a4f0aa736b4cb7c6 2024-11-14T02:44:32,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/1185202aa0e348649e054ed991f7d727 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/1185202aa0e348649e054ed991f7d727 2024-11-14T02:44:32,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/c46ddbbdfc6d440a8bbf5f3fcd9e51cd to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/c46ddbbdfc6d440a8bbf5f3fcd9e51cd 2024-11-14T02:44:32,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d436d32a42af49429b430297f25f2cf0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d436d32a42af49429b430297f25f2cf0 2024-11-14T02:44:32,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/1a5ac89bd6584dde9b661b02bb3e47d0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/1a5ac89bd6584dde9b661b02bb3e47d0 2024-11-14T02:44:32,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d06b702d67c34117aae5fcdc6dd560a8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d06b702d67c34117aae5fcdc6dd560a8 2024-11-14T02:44:32,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/ac75e8708bd04df1bb4f5518640be40b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/ac75e8708bd04df1bb4f5518640be40b 2024-11-14T02:44:32,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/33404f14b4774e11987b69e7901ff457 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/33404f14b4774e11987b69e7901ff457 2024-11-14T02:44:32,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/2a3986c8b9d5402f80e30ef470016b8e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/2a3986c8b9d5402f80e30ef470016b8e 2024-11-14T02:44:32,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/658077da9cb04c298fcd1c79ddd05199 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/658077da9cb04c298fcd1c79ddd05199 2024-11-14T02:44:32,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/dfd65ad2095b43c0bfbefb1b24067fed to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/dfd65ad2095b43c0bfbefb1b24067fed 2024-11-14T02:44:32,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d628e365ca7b41ec9487770c8aa22187 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d628e365ca7b41ec9487770c8aa22187 2024-11-14T02:44:32,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d1bc939ebda947ed8ed6accc70961ef6 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/d1bc939ebda947ed8ed6accc70961ef6 2024-11-14T02:44:32,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/bab7d0fc3682459c9ac1c6669d78f8cb to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/bab7d0fc3682459c9ac1c6669d78f8cb 2024-11-14T02:44:32,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/13df687fd22446d2901c10e0fb7ce7d7 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/13df687fd22446d2901c10e0fb7ce7d7 2024-11-14T02:44:32,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e03fdd9f02744cb1bd74590c43d9972e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e947872ac8764563b2c4a0dc978ba122, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/1b9f9e3ff12249dbb9d1a9bbd36abb88, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/b5813955a23e48f5905e5114a1914db3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/b85396eee9264722afa794e430e8cdfb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/23720e9a0f1c40c886b5c0dd7556f5db, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/100653382260487e9aac89d7df73080a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/efcbd1b177cb4825bbd8c8c5f394420d, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/0695495ac02249e382be884da73f71b3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/8f869c58e0d64a52af521812457fa721, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/cd3bfb18d86f4814ad4c8827a01ba4a2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/5abb77cb173742e78cd04fedc7335303, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/eae20ef9f8ad48029fb15e916d5a193d, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e23f9ab1414d42299e8a03526878d696, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/9bbefb9ed1b243458d033030be3e7899, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/ec1f47e0cacd4eeaa4253eb2493e0c00, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/a219ca1f535040069cda282328f901d7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/9356adbf8db842c8a7e4d67be1447da4] to archive 2024-11-14T02:44:32,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:44:32,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e03fdd9f02744cb1bd74590c43d9972e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e03fdd9f02744cb1bd74590c43d9972e 2024-11-14T02:44:32,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e947872ac8764563b2c4a0dc978ba122 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e947872ac8764563b2c4a0dc978ba122 2024-11-14T02:44:32,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/1b9f9e3ff12249dbb9d1a9bbd36abb88 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/1b9f9e3ff12249dbb9d1a9bbd36abb88 2024-11-14T02:44:32,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/b5813955a23e48f5905e5114a1914db3 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/b5813955a23e48f5905e5114a1914db3 2024-11-14T02:44:32,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/b85396eee9264722afa794e430e8cdfb to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/b85396eee9264722afa794e430e8cdfb 2024-11-14T02:44:32,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/23720e9a0f1c40c886b5c0dd7556f5db to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/23720e9a0f1c40c886b5c0dd7556f5db 2024-11-14T02:44:32,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/100653382260487e9aac89d7df73080a to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/100653382260487e9aac89d7df73080a 2024-11-14T02:44:32,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/efcbd1b177cb4825bbd8c8c5f394420d to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/efcbd1b177cb4825bbd8c8c5f394420d 2024-11-14T02:44:32,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/0695495ac02249e382be884da73f71b3 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/0695495ac02249e382be884da73f71b3 2024-11-14T02:44:32,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/8f869c58e0d64a52af521812457fa721 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/8f869c58e0d64a52af521812457fa721 2024-11-14T02:44:32,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/cd3bfb18d86f4814ad4c8827a01ba4a2 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/cd3bfb18d86f4814ad4c8827a01ba4a2 2024-11-14T02:44:32,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/5abb77cb173742e78cd04fedc7335303 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/5abb77cb173742e78cd04fedc7335303 2024-11-14T02:44:32,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/eae20ef9f8ad48029fb15e916d5a193d to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/eae20ef9f8ad48029fb15e916d5a193d 2024-11-14T02:44:32,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e23f9ab1414d42299e8a03526878d696 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/e23f9ab1414d42299e8a03526878d696 2024-11-14T02:44:32,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/9bbefb9ed1b243458d033030be3e7899 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/9bbefb9ed1b243458d033030be3e7899 2024-11-14T02:44:32,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/ec1f47e0cacd4eeaa4253eb2493e0c00 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/ec1f47e0cacd4eeaa4253eb2493e0c00 2024-11-14T02:44:32,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/a219ca1f535040069cda282328f901d7 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/a219ca1f535040069cda282328f901d7 2024-11-14T02:44:32,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/9356adbf8db842c8a7e4d67be1447da4 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/9356adbf8db842c8a7e4d67be1447da4 2024-11-14T02:44:32,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/d8398b46e3304b3b826332880b069750, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/78cd95cd3476456788f02d56e2817cb0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/b6c6bb1fb1334fe2b1a986fa2064a91a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/00aff73f35f140088b7f02da9a1594b9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/e3e87287c951440e95763bad46839de8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/3ba7801d213a4576b15cb09fb4a5fd22, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/4fd6f721b1af49a092ba68b48b596a50, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/8c2521161aa04a26b1d116a98751b9db, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/4f98bfe7adfe4f56bedd968d12470421, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/f4eaab2b00484a6193d8721220a41884, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/c5088952f88549968602783ec76e6dfe, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/aaee05883d9849c694f697661973904b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/85893e2a520b4a0a9d308b4981e7b253, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/04e5ae35655144bcb86584d1285cd275, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/c2dfda81a014455b9aea746ed6576f0e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/fa397c3c071a4601a2f472b1c3d358e1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/777570e0df7948d8af0ce4dffd6b2fe1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/36b750b797574fbfbaa53d7bfbf7df34] to archive 2024-11-14T02:44:32,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:44:32,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/d8398b46e3304b3b826332880b069750 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/d8398b46e3304b3b826332880b069750 2024-11-14T02:44:32,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/78cd95cd3476456788f02d56e2817cb0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/78cd95cd3476456788f02d56e2817cb0 2024-11-14T02:44:32,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/b6c6bb1fb1334fe2b1a986fa2064a91a to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/b6c6bb1fb1334fe2b1a986fa2064a91a 2024-11-14T02:44:32,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/00aff73f35f140088b7f02da9a1594b9 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/00aff73f35f140088b7f02da9a1594b9 2024-11-14T02:44:32,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/e3e87287c951440e95763bad46839de8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/e3e87287c951440e95763bad46839de8 2024-11-14T02:44:32,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/3ba7801d213a4576b15cb09fb4a5fd22 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/3ba7801d213a4576b15cb09fb4a5fd22 2024-11-14T02:44:32,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/4fd6f721b1af49a092ba68b48b596a50 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/4fd6f721b1af49a092ba68b48b596a50 2024-11-14T02:44:32,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/8c2521161aa04a26b1d116a98751b9db to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/8c2521161aa04a26b1d116a98751b9db 2024-11-14T02:44:32,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/4f98bfe7adfe4f56bedd968d12470421 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/4f98bfe7adfe4f56bedd968d12470421 2024-11-14T02:44:32,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/f4eaab2b00484a6193d8721220a41884 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/f4eaab2b00484a6193d8721220a41884 2024-11-14T02:44:32,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/c5088952f88549968602783ec76e6dfe to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/c5088952f88549968602783ec76e6dfe 2024-11-14T02:44:32,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/aaee05883d9849c694f697661973904b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/aaee05883d9849c694f697661973904b 2024-11-14T02:44:32,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/85893e2a520b4a0a9d308b4981e7b253 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/85893e2a520b4a0a9d308b4981e7b253 2024-11-14T02:44:32,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/04e5ae35655144bcb86584d1285cd275 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/04e5ae35655144bcb86584d1285cd275 2024-11-14T02:44:32,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/c2dfda81a014455b9aea746ed6576f0e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/c2dfda81a014455b9aea746ed6576f0e 2024-11-14T02:44:32,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/fa397c3c071a4601a2f472b1c3d358e1 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/fa397c3c071a4601a2f472b1c3d358e1 2024-11-14T02:44:32,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/777570e0df7948d8af0ce4dffd6b2fe1 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/777570e0df7948d8af0ce4dffd6b2fe1 2024-11-14T02:44:32,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/36b750b797574fbfbaa53d7bfbf7df34 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/36b750b797574fbfbaa53d7bfbf7df34 2024-11-14T02:44:32,737 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/recovered.edits/429.seqid, newMaxSeqId=429, maxSeqId=4 2024-11-14T02:44:32,738 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. 2024-11-14T02:44:32,738 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for 1d50bec2e36af6979b333798612ed279: Waiting for close lock at 1731552269389Running coprocessor pre-close hooks at 1731552269389Disabling compacts and flushes for region at 1731552269389Disabling writes for close at 1731552270984 (+1595 ms)Obtaining lock to block concurrent updates at 1731552270984Preparing flush snapshotting stores in 1d50bec2e36af6979b333798612ed279 at 1731552270984Finished memstore snapshotting TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731552270985 (+1 ms)Flushing stores of TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279. at 1731552270985Flushing 1d50bec2e36af6979b333798612ed279/A: creating writer at 1731552270985Flushing mob file 1d50bec2e36af6979b333798612ed279/A: appending metadata at 1731552270989 (+4 ms)Flushing mob file 1d50bec2e36af6979b333798612ed279/A: closing flushed file at 1731552270989Flushing 1d50bec2e36af6979b333798612ed279/A: appending metadata at 1731552271404 (+415 ms)Flushing 1d50bec2e36af6979b333798612ed279/A: closing flushed file at 1731552271404Flushing 1d50bec2e36af6979b333798612ed279/B: creating writer at 1731552271821 (+417 ms)Flushing 1d50bec2e36af6979b333798612ed279/B: appending metadata at 1731552271824 (+3 ms)Flushing 1d50bec2e36af6979b333798612ed279/B: closing flushed file at 1731552271824Flushing 1d50bec2e36af6979b333798612ed279/C: creating writer at 1731552272235 (+411 ms)Flushing 1d50bec2e36af6979b333798612ed279/C: appending metadata at 1731552272237 (+2 ms)Flushing 1d50bec2e36af6979b333798612ed279/C: closing flushed file at 1731552272237Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ebb87e8: reopening flushed file at 1731552272650 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32e1dacc: reopening flushed file at 1731552272659 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ad56d9a: reopening flushed file at 1731552272666 (+7 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 1d50bec2e36af6979b333798612ed279 in 1688ms, sequenceid=426, compaction requested=true at 1731552272673 (+7 ms)Writing region close event to WAL at 1731552272734 (+61 ms)Running coprocessor post-close hooks at 1731552272737 (+3 ms)Closed at 1731552272738 (+1 ms) 2024-11-14T02:44:32,739 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed 1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:32,740 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=1d50bec2e36af6979b333798612ed279, regionState=CLOSED 2024-11-14T02:44:32,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:44:32,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-14T02:44:32,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure 1d50bec2e36af6979b333798612ed279, server=e5dc8b5cf5e9,46387,1731552206828 in 3.5060 sec 2024-11-14T02:44:32,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-14T02:44:32,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d50bec2e36af6979b333798612ed279, UNASSIGN in 3.5110 sec 2024-11-14T02:44:32,747 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-14T02:44:32,747 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.5140 sec 2024-11-14T02:44:32,748 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552272748"}]},"ts":"1731552272748"} 2024-11-14T02:44:32,749 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T02:44:32,750 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T02:44:32,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.5230 sec 2024-11-14T02:44:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T02:44:33,386 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:33,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T02:44:33,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:33,392 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:33,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T02:44:33,393 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:33,397 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,400 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/recovered.edits] 2024-11-14T02:44:33,405 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/01fc9030cbee42e6bf1cb298d176c986 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/01fc9030cbee42e6bf1cb298d176c986 2024-11-14T02:44:33,408 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/181c26e9ac8b4aafa7dd2f3af3817afd to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/181c26e9ac8b4aafa7dd2f3af3817afd 2024-11-14T02:44:33,409 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/675dcb74204249c1950e7c45d532685f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/675dcb74204249c1950e7c45d532685f 2024-11-14T02:44:33,410 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/76cbe1ff29e745b7b154de0fbcd98d71 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/76cbe1ff29e745b7b154de0fbcd98d71 2024-11-14T02:44:33,411 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/e7b7d6941d064a2f8a59c34b5b00b501 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/e7b7d6941d064a2f8a59c34b5b00b501 2024-11-14T02:44:33,412 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/e8f0ce6dbcd443189010d4d1d431d90d to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/e8f0ce6dbcd443189010d4d1d431d90d 2024-11-14T02:44:33,413 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/fec8c6d7c424445e94246e9a7f6e6727 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/A/fec8c6d7c424445e94246e9a7f6e6727 2024-11-14T02:44:33,416 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/0959dbb0e8084ffea6d306f78700649c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/0959dbb0e8084ffea6d306f78700649c 2024-11-14T02:44:33,417 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/106a91ddd8b645f0a9b3022c80eeb718 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/106a91ddd8b645f0a9b3022c80eeb718 2024-11-14T02:44:33,418 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/1323d3ebfca543ae8b1428500376a198 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/1323d3ebfca543ae8b1428500376a198 2024-11-14T02:44:33,419 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/2c8fa969a0884f658bf4db4a721b7ae5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/2c8fa969a0884f658bf4db4a721b7ae5 2024-11-14T02:44:33,420 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/3c599224e7dd4c7fba949626c711b04b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/3c599224e7dd4c7fba949626c711b04b 2024-11-14T02:44:33,421 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/6c7dd43e29374c66911d0fa072920a3e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/6c7dd43e29374c66911d0fa072920a3e 2024-11-14T02:44:33,422 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/fbbdca010c0d4a0ca9b7e975ff64073d to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/B/fbbdca010c0d4a0ca9b7e975ff64073d 2024-11-14T02:44:33,424 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/6c1bdd053ab5424bad740101867148df to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/6c1bdd053ab5424bad740101867148df 2024-11-14T02:44:33,425 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/7c895a759a81401ab64b718523be994d to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/7c895a759a81401ab64b718523be994d 2024-11-14T02:44:33,426 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/7f3da944763d4f49ace362184ca56963 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/7f3da944763d4f49ace362184ca56963 2024-11-14T02:44:33,427 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/8272cf8548c144089b49502cbb7f6dc2 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/8272cf8548c144089b49502cbb7f6dc2 2024-11-14T02:44:33,428 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/963fe027ce2447b6b0430753baf9b4f7 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/963fe027ce2447b6b0430753baf9b4f7 2024-11-14T02:44:33,429 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/a907951a5fb7484791e4c00b41cbccba to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/a907951a5fb7484791e4c00b41cbccba 2024-11-14T02:44:33,430 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/ad7abcf8f7354e0aa57317a7855dc114 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/C/ad7abcf8f7354e0aa57317a7855dc114 2024-11-14T02:44:33,432 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/recovered.edits/429.seqid to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279/recovered.edits/429.seqid 2024-11-14T02:44:33,433 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,433 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T02:44:33,433 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T02:44:33,434 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T02:44:33,437 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142833ac16e08940e19f8dddb5b2157bb2_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142833ac16e08940e19f8dddb5b2157bb2_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,438 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111430df3170ee0f44ddbcb26e38ed0cd5fc_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111430df3170ee0f44ddbcb26e38ed0cd5fc_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,439 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143f94dddb45944faa83e32edeb616aa46_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143f94dddb45944faa83e32edeb616aa46_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,440 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144418a2dc9d4c4ee984af0eb5f2ead2bb_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144418a2dc9d4c4ee984af0eb5f2ead2bb_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,441 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146a84d8eb1a51408593fd9609454ab20b_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146a84d8eb1a51408593fd9609454ab20b_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,442 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147b1b5cf4ae3e4a2589f53a7118be6b3e_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147b1b5cf4ae3e4a2589f53a7118be6b3e_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,443 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149be1a3758b2c461697ce22ae30f40cd0_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149be1a3758b2c461697ce22ae30f40cd0_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,445 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a6284526233a456c933268737ae758f0_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a6284526233a456c933268737ae758f0_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,446 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b607fbe446fc445da2c8dc47f98620a4_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b607fbe446fc445da2c8dc47f98620a4_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,447 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b780b1929d504b189a009a977e7518db_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b780b1929d504b189a009a977e7518db_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,448 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c243b880a4714fb08a40742915017cb1_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c243b880a4714fb08a40742915017cb1_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,449 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c4d010cbb6f74aa49484a0972050f0f6_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c4d010cbb6f74aa49484a0972050f0f6_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,450 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d09e32e1df5346229fd2ae09eb319c93_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d09e32e1df5346229fd2ae09eb319c93_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,451 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d8a084286ffd49e3aa06a26504327e28_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d8a084286ffd49e3aa06a26504327e28_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,453 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114db7bbad4d767476cbc143040a5727341_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114db7bbad4d767476cbc143040a5727341_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,454 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e61e97c7366f4e5380379c3d7c6f1d6f_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e61e97c7366f4e5380379c3d7c6f1d6f_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,455 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ed247f869dde4fb585aae51f190e0267_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ed247f869dde4fb585aae51f190e0267_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,456 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f425e1979d1e46b7a7c28341e3aaf4d7_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f425e1979d1e46b7a7c28341e3aaf4d7_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,457 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fa176405333c45679212d2a44f80adb2_1d50bec2e36af6979b333798612ed279 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fa176405333c45679212d2a44f80adb2_1d50bec2e36af6979b333798612ed279 2024-11-14T02:44:33,457 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T02:44:33,459 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:33,462 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T02:44:33,463 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T02:44:33,465 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:33,465 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T02:44:33,465 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731552273465"}]},"ts":"9223372036854775807"} 2024-11-14T02:44:33,467 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T02:44:33,467 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1d50bec2e36af6979b333798612ed279, NAME => 'TestAcidGuarantees,,1731552244910.1d50bec2e36af6979b333798612ed279.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T02:44:33,467 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T02:44:33,467 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731552273467"}]},"ts":"9223372036854775807"} 2024-11-14T02:44:33,468 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T02:44:33,469 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:33,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 81 msec 2024-11-14T02:44:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T02:44:33,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T02:44:33,496 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:33,504 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-242810252_22 at /127.0.0.1:55274 [Waiting for operation #294] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/cluster_d5add8ed-0ad9-3315-92f0-5e7efe2147ab/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-521717827_22 at /127.0.0.1:50302 [Waiting for operation #142] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-521717827_22 at /127.0.0.1:55182 [Waiting for operation #316] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-242810252_22 at /127.0.0.1:50354 [Waiting for operation #117] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/cluster_d5add8ed-0ad9-3315-92f0-5e7efe2147ab/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 383) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9977 (was 10077) 2024-11-14T02:44:33,514 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=9977 2024-11-14T02:44:33,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:44:33,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:44:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T02:44:33,518 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T02:44:33,518 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:33,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-14T02:44:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-14T02:44:33,519 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T02:44:33,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742044_1220 (size=1061) 2024-11-14T02:44:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-14T02:44:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-14T02:44:33,932 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1 2024-11-14T02:44:33,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742045_1221 (size=53) 2024-11-14T02:44:34,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-14T02:44:34,343 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:44:34,344 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 21da8e2d40bdcddc538290150d14efc8, disabling compactions & flushes 2024-11-14T02:44:34,344 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:34,344 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:34,344 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. after waiting 0 ms 2024-11-14T02:44:34,344 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:34,344 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:34,344 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 21da8e2d40bdcddc538290150d14efc8: Waiting for close lock at 1731552274344Disabling compacts and flushes for region at 1731552274344Disabling writes for close at 1731552274344Writing region close event to WAL at 1731552274344Closed at 1731552274344 2024-11-14T02:44:34,346 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T02:44:34,347 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731552274346"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731552274346"}]},"ts":"1731552274346"} 2024-11-14T02:44:34,350 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T02:44:34,352 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T02:44:34,352 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552274352"}]},"ts":"1731552274352"} 2024-11-14T02:44:34,354 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T02:44:34,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21da8e2d40bdcddc538290150d14efc8, ASSIGN}] 2024-11-14T02:44:34,356 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21da8e2d40bdcddc538290150d14efc8, ASSIGN 2024-11-14T02:44:34,357 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21da8e2d40bdcddc538290150d14efc8, ASSIGN; state=OFFLINE, location=e5dc8b5cf5e9,46387,1731552206828; forceNewPlan=false, retain=false 2024-11-14T02:44:34,508 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=21da8e2d40bdcddc538290150d14efc8, regionState=OPENING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:34,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21da8e2d40bdcddc538290150d14efc8, ASSIGN because future has completed 2024-11-14T02:44:34,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:44:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-14T02:44:34,665 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:34,665 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:44:34,666 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:34,666 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:44:34,666 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:34,666 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:34,667 INFO [StoreOpener-21da8e2d40bdcddc538290150d14efc8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:34,668 INFO [StoreOpener-21da8e2d40bdcddc538290150d14efc8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:44:34,668 INFO [StoreOpener-21da8e2d40bdcddc538290150d14efc8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21da8e2d40bdcddc538290150d14efc8 columnFamilyName A 2024-11-14T02:44:34,668 DEBUG [StoreOpener-21da8e2d40bdcddc538290150d14efc8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:34,669 INFO [StoreOpener-21da8e2d40bdcddc538290150d14efc8-1 {}] regionserver.HStore(327): Store=21da8e2d40bdcddc538290150d14efc8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:34,669 INFO [StoreOpener-21da8e2d40bdcddc538290150d14efc8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:34,670 INFO [StoreOpener-21da8e2d40bdcddc538290150d14efc8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:44:34,670 INFO [StoreOpener-21da8e2d40bdcddc538290150d14efc8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21da8e2d40bdcddc538290150d14efc8 columnFamilyName B 2024-11-14T02:44:34,670 DEBUG [StoreOpener-21da8e2d40bdcddc538290150d14efc8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:34,670 INFO [StoreOpener-21da8e2d40bdcddc538290150d14efc8-1 {}] regionserver.HStore(327): Store=21da8e2d40bdcddc538290150d14efc8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:34,670 INFO [StoreOpener-21da8e2d40bdcddc538290150d14efc8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:34,671 INFO [StoreOpener-21da8e2d40bdcddc538290150d14efc8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:44:34,672 INFO [StoreOpener-21da8e2d40bdcddc538290150d14efc8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21da8e2d40bdcddc538290150d14efc8 columnFamilyName C 2024-11-14T02:44:34,672 DEBUG [StoreOpener-21da8e2d40bdcddc538290150d14efc8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:44:34,672 INFO [StoreOpener-21da8e2d40bdcddc538290150d14efc8-1 {}] regionserver.HStore(327): Store=21da8e2d40bdcddc538290150d14efc8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:44:34,672 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:34,672 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:34,673 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:34,674 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:34,675 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:34,675 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:34,675 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:44:34,676 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:34,678 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:44:34,679 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 21da8e2d40bdcddc538290150d14efc8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61997748, jitterRate=-0.07616156339645386}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:44:34,679 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:34,679 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 21da8e2d40bdcddc538290150d14efc8: Running coprocessor pre-open hook at 1731552274666Writing region info on filesystem at 1731552274666Initializing all the Stores at 1731552274666Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552274667 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552274667Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552274667Cleaning up temporary data from old regions at 1731552274675 (+8 ms)Running coprocessor post-open hooks at 1731552274679 (+4 ms)Region opened successfully at 1731552274679 2024-11-14T02:44:34,680 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., pid=59, masterSystemTime=1731552274662 2024-11-14T02:44:34,682 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:34,682 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:34,682 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=21da8e2d40bdcddc538290150d14efc8, regionState=OPEN, openSeqNum=2, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:34,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:44:34,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-14T02:44:34,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure 21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 in 174 msec 2024-11-14T02:44:34,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-14T02:44:34,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21da8e2d40bdcddc538290150d14efc8, ASSIGN in 330 msec 2024-11-14T02:44:34,687 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T02:44:34,687 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552274687"}]},"ts":"1731552274687"} 2024-11-14T02:44:34,689 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T02:44:34,689 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T02:44:34,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1730 sec 2024-11-14T02:44:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-14T02:44:35,666 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:35,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8da64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:35,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:35,668 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4efe1c64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:35,670 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:35,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fb966be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,671 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:35,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:35,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:35,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:35,674 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:44:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:44:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@92e36a5 2024-11-14T02:44:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:44:35,676 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:44:35,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:35,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:35,679 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:35,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:35,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:35,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b330f4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:35,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:35,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:35,680 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:35,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b2ac1b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:35,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:35,683 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49ffbff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:35,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:35,685 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:35,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd15c7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:35,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:35,687 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:35,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:35,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:35,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2662aa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:35,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:35,689 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:35,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e91984f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:35,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:35,692 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a20508e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:35,694 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@133afd88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:35,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:35,697 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd5b996, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:35,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:35,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:35,699 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:35,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56271d14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,700 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:35,700 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:35,701 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:35,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bd6b64b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:35,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:35,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:35,703 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:35,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a8751d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:35,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:35,705 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@233cb3be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:35,706 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@92dea22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:35,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:35,708 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:35,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:35,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@676b6df2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:35,710 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:35,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45120b19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,711 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:35,711 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:35,712 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ee23c7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:35,713 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300fa89a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40916ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:44:35,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:44:35,715 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:44:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:44:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:44:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c7567b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:44:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:44:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:35,716 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:44:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e9d046, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:44:35,717 DEBUG [Thread-1060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:35,717 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:35,717 DEBUG [Thread-1058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:35,717 DEBUG [Thread-1049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:35,717 DEBUG [Thread-1045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:35,718 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:35,719 DEBUG [Thread-1051 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:35,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:35,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:35,719 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:35,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:35,720 DEBUG [Thread-1049 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:35,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:35,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:35,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:44:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:44:35,722 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:35,722 DEBUG [Thread-1062 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:44:35,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:44:35,723 DEBUG [Thread-1043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:35,723 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:35,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:44:35,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:44:35,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:35,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:35,725 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:35,725 DEBUG [Thread-1047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:44:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:44:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:44:35,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:35,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:44:35,727 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:44:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:44:35,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:44:35,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:44:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-14T02:44:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T02:44:35,767 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:35,769 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:35,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:35,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:44:35,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:35,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:35,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:35,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:35,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:35,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:35,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1731552335796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:35,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:35,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552275787, max timestamp=1731552275799Num uniques 859; 2024-11-14T02:44:35,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552275787, max timestamp=1731552275799Num uniques 859; 2024-11-14T02:44:35,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552275787, max timestamp=1731552275799Num uniques 859; 2024-11-14T02:44:35,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:35,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:35,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:35,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49758 deadline: 1731552335796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:35,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/7585ab9dc25e4e8ab304b145211db174 is 50, key is test_row_0/A:col10/1731552275766/Put/seqid=0 2024-11-14T02:44:35,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742046_1222 (size=12001) 2024-11-14T02:44:35,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552335824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:35,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:35,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552335830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:35,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552335831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:35,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T02:44:35,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1731552335905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:35,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:35,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49758 deadline: 1731552335905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:35,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:35,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:35,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:35,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:35,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:35,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:35,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:35,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:35,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552335935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:35,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:35,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552335945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:35,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552335945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:36,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:36,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:36,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:36,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:36,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:36,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T02:44:36,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1731552336115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:36,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49758 deadline: 1731552336115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:36,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552336145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:36,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:36,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552336155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:36,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552336155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:36,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/7585ab9dc25e4e8ab304b145211db174 2024-11-14T02:44:36,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:36,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:36,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:36,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:36,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:36,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:36,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/355ec4b861bc4fbe8c5e9e257843cb1e is 50, key is test_row_0/B:col10/1731552275766/Put/seqid=0 2024-11-14T02:44:36,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742047_1223 (size=12001) 2024-11-14T02:44:36,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/355ec4b861bc4fbe8c5e9e257843cb1e 2024-11-14T02:44:36,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/ebf9cfc3fb454581bacef027ead29d73 is 50, key is test_row_0/C:col10/1731552275766/Put/seqid=0 2024-11-14T02:44:36,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742048_1224 (size=12001) 2024-11-14T02:44:36,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/ebf9cfc3fb454581bacef027ead29d73 2024-11-14T02:44:36,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/7585ab9dc25e4e8ab304b145211db174 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/7585ab9dc25e4e8ab304b145211db174 2024-11-14T02:44:36,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/7585ab9dc25e4e8ab304b145211db174, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T02:44:36,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/355ec4b861bc4fbe8c5e9e257843cb1e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/355ec4b861bc4fbe8c5e9e257843cb1e 2024-11-14T02:44:36,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/355ec4b861bc4fbe8c5e9e257843cb1e, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T02:44:36,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/ebf9cfc3fb454581bacef027ead29d73 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/ebf9cfc3fb454581bacef027ead29d73 2024-11-14T02:44:36,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/ebf9cfc3fb454581bacef027ead29d73, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T02:44:36,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 21da8e2d40bdcddc538290150d14efc8 in 524ms, sequenceid=12, compaction requested=false 2024-11-14T02:44:36,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:36,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T02:44:36,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:36,387 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-14T02:44:36,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:36,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:36,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:36,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:36,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:36,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/5548fe3cfcb74869bf6cd4ab53d0157c is 50, key is test_row_0/A:col10/1731552275829/Put/seqid=0 2024-11-14T02:44:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T02:44:36,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742049_1225 (size=12001) 2024-11-14T02:44:36,398 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/5548fe3cfcb74869bf6cd4ab53d0157c 2024-11-14T02:44:36,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/007e9cdcf3d24adf8271a5db9693c1cb is 50, key is test_row_0/B:col10/1731552275829/Put/seqid=0 2024-11-14T02:44:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:36,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:36,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742050_1226 (size=12001) 2024-11-14T02:44:36,429 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/007e9cdcf3d24adf8271a5db9693c1cb 2024-11-14T02:44:36,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/62e35067959d4195b8e9fe91627c4f00 is 50, key is test_row_0/C:col10/1731552275829/Put/seqid=0 2024-11-14T02:44:36,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1731552336439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:36,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742051_1227 (size=12001) 2024-11-14T02:44:36,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49758 deadline: 1731552336442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:36,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552336455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:36,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552336465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:36,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552336465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:36,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:36,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1731552336545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:36,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49758 deadline: 1731552336555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:36,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1731552336755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:36,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49758 deadline: 1731552336765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:36,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:36,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:36,846 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/62e35067959d4195b8e9fe91627c4f00 2024-11-14T02:44:36,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/5548fe3cfcb74869bf6cd4ab53d0157c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/5548fe3cfcb74869bf6cd4ab53d0157c 2024-11-14T02:44:36,854 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/5548fe3cfcb74869bf6cd4ab53d0157c, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T02:44:36,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/007e9cdcf3d24adf8271a5db9693c1cb as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/007e9cdcf3d24adf8271a5db9693c1cb 2024-11-14T02:44:36,859 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/007e9cdcf3d24adf8271a5db9693c1cb, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T02:44:36,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/62e35067959d4195b8e9fe91627c4f00 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/62e35067959d4195b8e9fe91627c4f00 2024-11-14T02:44:36,863 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/62e35067959d4195b8e9fe91627c4f00, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T02:44:36,864 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 21da8e2d40bdcddc538290150d14efc8 in 477ms, sequenceid=48, compaction requested=false 2024-11-14T02:44:36,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:36,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:36,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-14T02:44:36,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-14T02:44:36,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-14T02:44:36,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0960 sec 2024-11-14T02:44:36,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.1490 sec 2024-11-14T02:44:36,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T02:44:36,905 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:36,905 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:36,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-14T02:44:36,907 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:36,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T02:44:36,907 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:36,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:36,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:36,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T02:44:36,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:36,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:36,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:36,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:36,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:36,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:36,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/abb4ca26593d4910a51a556129029ef4 is 50, key is test_row_1/A:col10/1731552276966/Put/seqid=0 2024-11-14T02:44:36,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742052_1228 (size=9657) 2024-11-14T02:44:37,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731552276986, max timestamp=1731552276997Num uniques 859; 2024-11-14T02:44:37,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:44:37,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731552276986, max timestamp=1731552276997Num uniques 859; 2024-11-14T02:44:37,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:44:37,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731552276986, max timestamp=1731552276997Num uniques 859; 2024-11-14T02:44:37,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:44:37,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T02:44:37,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552337022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552337025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552337027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T02:44:37,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:37,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:37,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:37,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:37,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:37,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1731552337066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49758 deadline: 1731552337075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552337135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552337135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552337136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T02:44:37,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:37,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:37,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:37,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:37,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T02:44:37,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552337345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552337345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552337345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T02:44:37,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:37,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:37,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:37,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:37,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:37,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/abb4ca26593d4910a51a556129029ef4 2024-11-14T02:44:37,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/2294e862b6ab48a6aba3857006f76f43 is 50, key is test_row_1/B:col10/1731552276966/Put/seqid=0 2024-11-14T02:44:37,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742053_1229 (size=9657) 2024-11-14T02:44:37,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T02:44:37,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:37,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:37,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:37,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:37,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T02:44:37,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1731552337575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49758 deadline: 1731552337585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552337655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552337655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:37,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552337655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:37,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:37,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:37,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T02:44:37,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:37,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:37,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:37,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:37,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:37,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/2294e862b6ab48a6aba3857006f76f43 2024-11-14T02:44:37,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/5e9f065581cd43dfae7b38d9719d647c is 50, key is test_row_1/C:col10/1731552276966/Put/seqid=0 2024-11-14T02:44:37,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742054_1230 (size=9657) 2024-11-14T02:44:37,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/5e9f065581cd43dfae7b38d9719d647c 2024-11-14T02:44:37,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/abb4ca26593d4910a51a556129029ef4 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/abb4ca26593d4910a51a556129029ef4 2024-11-14T02:44:37,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/abb4ca26593d4910a51a556129029ef4, entries=100, sequenceid=59, filesize=9.4 K 2024-11-14T02:44:37,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/2294e862b6ab48a6aba3857006f76f43 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/2294e862b6ab48a6aba3857006f76f43 2024-11-14T02:44:37,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/2294e862b6ab48a6aba3857006f76f43, entries=100, sequenceid=59, filesize=9.4 K 2024-11-14T02:44:37,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/5e9f065581cd43dfae7b38d9719d647c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/5e9f065581cd43dfae7b38d9719d647c 2024-11-14T02:44:37,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/5e9f065581cd43dfae7b38d9719d647c, entries=100, sequenceid=59, filesize=9.4 K 2024-11-14T02:44:37,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T02:44:37,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 21da8e2d40bdcddc538290150d14efc8 in 856ms, sequenceid=59, compaction requested=true 2024-11-14T02:44:37,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:37,823 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-14T02:44:37,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:37,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:37,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:37,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:37,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:37,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:37,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:37,824 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:37,824 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:37,826 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:37,826 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/A is initiating minor compaction (all files) 2024-11-14T02:44:37,826 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/A in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:37,826 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/7585ab9dc25e4e8ab304b145211db174, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/5548fe3cfcb74869bf6cd4ab53d0157c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/abb4ca26593d4910a51a556129029ef4] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=32.9 K 2024-11-14T02:44:37,826 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:37,826 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/B is initiating minor compaction (all files) 2024-11-14T02:44:37,826 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/B in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:37,826 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/355ec4b861bc4fbe8c5e9e257843cb1e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/007e9cdcf3d24adf8271a5db9693c1cb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/2294e862b6ab48a6aba3857006f76f43] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=32.9 K 2024-11-14T02:44:37,827 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7585ab9dc25e4e8ab304b145211db174, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731552275766 2024-11-14T02:44:37,827 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5548fe3cfcb74869bf6cd4ab53d0157c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731552275822 2024-11-14T02:44:37,827 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 355ec4b861bc4fbe8c5e9e257843cb1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731552275766 2024-11-14T02:44:37,827 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 007e9cdcf3d24adf8271a5db9693c1cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731552275822 2024-11-14T02:44:37,827 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting abb4ca26593d4910a51a556129029ef4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552276441 2024-11-14T02:44:37,828 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 2294e862b6ab48a6aba3857006f76f43, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552276441 2024-11-14T02:44:37,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/f018de12f9a54f539b4c6148703fb578 is 50, key is test_row_0/A:col10/1731552277019/Put/seqid=0 2024-11-14T02:44:37,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742055_1231 (size=12001) 2024-11-14T02:44:37,850 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#B#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:37,850 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/8f3d88c51d054fbfb2352c11aed28494 is 50, key is test_row_0/B:col10/1731552275829/Put/seqid=0 2024-11-14T02:44:37,858 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#A#compaction#190 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:37,858 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/f7342d0b76ac4cd68e439aa5efe8c25b is 50, key is test_row_0/A:col10/1731552275829/Put/seqid=0 2024-11-14T02:44:37,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742056_1232 (size=12104) 2024-11-14T02:44:37,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742057_1233 (size=12104) 2024-11-14T02:44:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T02:44:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:38,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:38,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552338176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:38,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552338180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:38,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552338182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:38,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:38,243 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/f018de12f9a54f539b4c6148703fb578 2024-11-14T02:44:38,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/5726f86a6fe446d78069d8131f1e4379 is 50, key is test_row_0/B:col10/1731552277019/Put/seqid=0 2024-11-14T02:44:38,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742058_1234 (size=12001) 2024-11-14T02:44:38,265 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T02:44:38,269 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/8f3d88c51d054fbfb2352c11aed28494 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/8f3d88c51d054fbfb2352c11aed28494 2024-11-14T02:44:38,279 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/B of 21da8e2d40bdcddc538290150d14efc8 into 8f3d88c51d054fbfb2352c11aed28494(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:38,279 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:38,279 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/B, priority=13, startTime=1731552277824; duration=0sec 2024-11-14T02:44:38,279 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:38,279 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:B 2024-11-14T02:44:38,279 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:38,283 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:38,283 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/C is initiating minor compaction (all files) 2024-11-14T02:44:38,284 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/C in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:38,286 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/ebf9cfc3fb454581bacef027ead29d73, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/62e35067959d4195b8e9fe91627c4f00, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/5e9f065581cd43dfae7b38d9719d647c] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=32.9 K 2024-11-14T02:44:38,287 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/f7342d0b76ac4cd68e439aa5efe8c25b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f7342d0b76ac4cd68e439aa5efe8c25b 2024-11-14T02:44:38,287 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting ebf9cfc3fb454581bacef027ead29d73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731552275766 2024-11-14T02:44:38,288 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 62e35067959d4195b8e9fe91627c4f00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731552275822 2024-11-14T02:44:38,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552338285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:38,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:38,290 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e9f065581cd43dfae7b38d9719d647c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552276441 2024-11-14T02:44:38,293 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/A of 21da8e2d40bdcddc538290150d14efc8 into f7342d0b76ac4cd68e439aa5efe8c25b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:38,294 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:38,294 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/A, priority=13, startTime=1731552277823; duration=0sec 2024-11-14T02:44:38,294 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:38,294 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:A 2024-11-14T02:44:38,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552338295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:38,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552338295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:38,301 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#C#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:38,301 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/3c67a92505894acbb5ce20f1e5acd42e is 50, key is test_row_0/C:col10/1731552275829/Put/seqid=0 2024-11-14T02:44:38,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742059_1235 (size=12104) 2024-11-14T02:44:38,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552338495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:38,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:38,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552338505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552338505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:38,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1731552338585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:38,588 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in 21da8e2d40bdcddc538290150d14efc8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:38,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49758 deadline: 1731552338595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:38,599 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in 21da8e2d40bdcddc538290150d14efc8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:38,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:38,654 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/5726f86a6fe446d78069d8131f1e4379 2024-11-14T02:44:38,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/815ddd9208e84d379ce18de49ae85d70 is 50, key is test_row_0/C:col10/1731552277019/Put/seqid=0 2024-11-14T02:44:38,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742060_1236 (size=12001) 2024-11-14T02:44:38,710 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/3c67a92505894acbb5ce20f1e5acd42e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3c67a92505894acbb5ce20f1e5acd42e 2024-11-14T02:44:38,715 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/C of 21da8e2d40bdcddc538290150d14efc8 into 3c67a92505894acbb5ce20f1e5acd42e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:38,715 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:38,715 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/C, priority=13, startTime=1731552277824; duration=0sec 2024-11-14T02:44:38,715 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:38,715 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:C 2024-11-14T02:44:38,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552338805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:38,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552338815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:38,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552338815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T02:44:39,070 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/815ddd9208e84d379ce18de49ae85d70 2024-11-14T02:44:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/f018de12f9a54f539b4c6148703fb578 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f018de12f9a54f539b4c6148703fb578 2024-11-14T02:44:39,077 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f018de12f9a54f539b4c6148703fb578, entries=150, sequenceid=93, filesize=11.7 K 2024-11-14T02:44:39,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/5726f86a6fe446d78069d8131f1e4379 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/5726f86a6fe446d78069d8131f1e4379 2024-11-14T02:44:39,082 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/5726f86a6fe446d78069d8131f1e4379, entries=150, sequenceid=93, filesize=11.7 K 2024-11-14T02:44:39,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/815ddd9208e84d379ce18de49ae85d70 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/815ddd9208e84d379ce18de49ae85d70 2024-11-14T02:44:39,088 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/815ddd9208e84d379ce18de49ae85d70, entries=150, sequenceid=93, filesize=11.7 K 2024-11-14T02:44:39,088 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 21da8e2d40bdcddc538290150d14efc8 in 1265ms, sequenceid=93, compaction requested=false 2024-11-14T02:44:39,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:39,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:39,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-14T02:44:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-14T02:44:39,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-14T02:44:39,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1830 sec 2024-11-14T02:44:39,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.1870 sec 2024-11-14T02:44:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:39,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T02:44:39,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:39,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:39,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:39,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:39,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:39,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:39,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/4972db8175844514bd6c11dc3064522e is 50, key is test_row_0/A:col10/1731552279315/Put/seqid=0 2024-11-14T02:44:39,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742061_1237 (size=14341) 2024-11-14T02:44:39,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/4972db8175844514bd6c11dc3064522e 2024-11-14T02:44:39,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/27c3491380814481bd4c44945488a59a is 50, key is test_row_0/B:col10/1731552279315/Put/seqid=0 2024-11-14T02:44:39,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742062_1238 (size=12001) 2024-11-14T02:44:39,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/27c3491380814481bd4c44945488a59a 2024-11-14T02:44:39,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552339349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:39,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552339354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:39,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552339356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:39,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/0bfaa9bddfa34725a09bdc2e0634609d is 50, key is test_row_0/C:col10/1731552279315/Put/seqid=0 2024-11-14T02:44:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:39,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742063_1239 (size=12001) 2024-11-14T02:44:39,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552339465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:39,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552339465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:39,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552339465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:39,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552339675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:39,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552339675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:39,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552339675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:39,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/0bfaa9bddfa34725a09bdc2e0634609d 2024-11-14T02:44:39,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/4972db8175844514bd6c11dc3064522e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/4972db8175844514bd6c11dc3064522e 2024-11-14T02:44:39,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/4972db8175844514bd6c11dc3064522e, entries=200, sequenceid=108, filesize=14.0 K 2024-11-14T02:44:39,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/27c3491380814481bd4c44945488a59a as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/27c3491380814481bd4c44945488a59a 2024-11-14T02:44:39,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/27c3491380814481bd4c44945488a59a, entries=150, sequenceid=108, filesize=11.7 K 2024-11-14T02:44:39,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/0bfaa9bddfa34725a09bdc2e0634609d as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/0bfaa9bddfa34725a09bdc2e0634609d 2024-11-14T02:44:39,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/0bfaa9bddfa34725a09bdc2e0634609d, entries=150, sequenceid=108, filesize=11.7 K 2024-11-14T02:44:39,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 21da8e2d40bdcddc538290150d14efc8 in 475ms, sequenceid=108, compaction requested=true 2024-11-14T02:44:39,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:39,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:39,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:39,791 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:39,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:39,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:39,791 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:39,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:39,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:39,792 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:39,792 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:39,792 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/A is initiating minor compaction (all files) 2024-11-14T02:44:39,792 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/B is initiating minor compaction (all files) 2024-11-14T02:44:39,792 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/A in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:39,792 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/B in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:39,792 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f7342d0b76ac4cd68e439aa5efe8c25b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f018de12f9a54f539b4c6148703fb578, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/4972db8175844514bd6c11dc3064522e] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=37.5 K 2024-11-14T02:44:39,792 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/8f3d88c51d054fbfb2352c11aed28494, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/5726f86a6fe446d78069d8131f1e4379, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/27c3491380814481bd4c44945488a59a] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=35.3 K 2024-11-14T02:44:39,792 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7342d0b76ac4cd68e439aa5efe8c25b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552275829 2024-11-14T02:44:39,792 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f3d88c51d054fbfb2352c11aed28494, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552275829 2024-11-14T02:44:39,793 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting f018de12f9a54f539b4c6148703fb578, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731552277019 2024-11-14T02:44:39,793 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 5726f86a6fe446d78069d8131f1e4379, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731552277019 2024-11-14T02:44:39,793 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 27c3491380814481bd4c44945488a59a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731552278178 2024-11-14T02:44:39,793 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4972db8175844514bd6c11dc3064522e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731552278174 2024-11-14T02:44:39,799 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#B#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:39,800 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/05a246e0cc1b45399b276550d0ae9286 is 50, key is test_row_0/B:col10/1731552279315/Put/seqid=0 2024-11-14T02:44:39,800 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#A#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:39,800 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/e170a5eb5fc44198864f52402ddd2bde is 50, key is test_row_0/A:col10/1731552279315/Put/seqid=0 2024-11-14T02:44:39,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742064_1240 (size=12207) 2024-11-14T02:44:39,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742065_1241 (size=12207) 2024-11-14T02:44:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:39,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T02:44:39,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:39,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:39,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:39,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:39,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:39,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:39,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/644651748b6a403bb049eaf289f5f29f is 50, key is test_row_0/A:col10/1731552279986/Put/seqid=0 2024-11-14T02:44:39,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742066_1242 (size=14391) 2024-11-14T02:44:40,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552340001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:40,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552340003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:40,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552340004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:40,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552340115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:40,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552340116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552340116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:40,215 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/05a246e0cc1b45399b276550d0ae9286 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/05a246e0cc1b45399b276550d0ae9286 2024-11-14T02:44:40,216 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/e170a5eb5fc44198864f52402ddd2bde as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e170a5eb5fc44198864f52402ddd2bde 2024-11-14T02:44:40,219 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/B of 21da8e2d40bdcddc538290150d14efc8 into 05a246e0cc1b45399b276550d0ae9286(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:40,219 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:40,219 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/B, priority=13, startTime=1731552279791; duration=0sec 2024-11-14T02:44:40,219 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:40,219 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:B 2024-11-14T02:44:40,219 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:40,221 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:40,221 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/C is initiating minor compaction (all files) 2024-11-14T02:44:40,221 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/C in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:40,221 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3c67a92505894acbb5ce20f1e5acd42e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/815ddd9208e84d379ce18de49ae85d70, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/0bfaa9bddfa34725a09bdc2e0634609d] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=35.3 K 2024-11-14T02:44:40,224 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c67a92505894acbb5ce20f1e5acd42e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552275829 2024-11-14T02:44:40,224 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 815ddd9208e84d379ce18de49ae85d70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731552277019 2024-11-14T02:44:40,225 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bfaa9bddfa34725a09bdc2e0634609d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731552278178 2024-11-14T02:44:40,226 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/A of 21da8e2d40bdcddc538290150d14efc8 into e170a5eb5fc44198864f52402ddd2bde(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:40,226 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:40,226 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/A, priority=13, startTime=1731552279791; duration=0sec 2024-11-14T02:44:40,226 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:40,226 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:A 2024-11-14T02:44:40,232 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#C#compaction#200 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:40,233 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/5ee4b821bd9b4c8fa3d5d2d226482ce3 is 50, key is test_row_0/C:col10/1731552279315/Put/seqid=0 2024-11-14T02:44:40,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742067_1243 (size=12207) 2024-11-14T02:44:40,242 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/5ee4b821bd9b4c8fa3d5d2d226482ce3 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/5ee4b821bd9b4c8fa3d5d2d226482ce3 2024-11-14T02:44:40,246 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/C of 21da8e2d40bdcddc538290150d14efc8 into 5ee4b821bd9b4c8fa3d5d2d226482ce3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:40,246 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:40,247 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/C, priority=13, startTime=1731552279791; duration=0sec 2024-11-14T02:44:40,247 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:40,247 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:C 2024-11-14T02:44:40,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:40,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552340325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552340326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:40,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552340326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:40,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/644651748b6a403bb049eaf289f5f29f 2024-11-14T02:44:40,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/4892a92fcd7f4020b7353cba0c2a0e55 is 50, key is test_row_0/B:col10/1731552279986/Put/seqid=0 2024-11-14T02:44:40,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742068_1244 (size=12051) 2024-11-14T02:44:40,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1731552340595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:40,597 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in 21da8e2d40bdcddc538290150d14efc8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:40,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49758 deadline: 1731552340615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:40,617 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in 21da8e2d40bdcddc538290150d14efc8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:40,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552340635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:40,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552340635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552340635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:40,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/4892a92fcd7f4020b7353cba0c2a0e55 2024-11-14T02:44:40,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/02db01f26d6649499d3e9c7f4053fe3f is 50, key is test_row_0/C:col10/1731552279986/Put/seqid=0 2024-11-14T02:44:40,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742069_1245 (size=12051) 2024-11-14T02:44:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T02:44:41,065 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:41,065 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-14T02:44:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T02:44:41,067 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:41,067 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:41,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:41,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552341145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:41,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552341145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:41,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552341145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:41,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:41,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:41,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:41,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T02:44:41,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T02:44:41,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:41,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:41,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:41,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:41,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:41,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/02db01f26d6649499d3e9c7f4053fe3f 2024-11-14T02:44:41,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/644651748b6a403bb049eaf289f5f29f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/644651748b6a403bb049eaf289f5f29f 2024-11-14T02:44:41,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/644651748b6a403bb049eaf289f5f29f, entries=200, sequenceid=129, filesize=14.1 K 2024-11-14T02:44:41,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/4892a92fcd7f4020b7353cba0c2a0e55 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/4892a92fcd7f4020b7353cba0c2a0e55 2024-11-14T02:44:41,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/4892a92fcd7f4020b7353cba0c2a0e55, entries=150, sequenceid=129, filesize=11.8 K 2024-11-14T02:44:41,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/02db01f26d6649499d3e9c7f4053fe3f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/02db01f26d6649499d3e9c7f4053fe3f 2024-11-14T02:44:41,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/02db01f26d6649499d3e9c7f4053fe3f, entries=150, sequenceid=129, filesize=11.8 K 2024-11-14T02:44:41,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 21da8e2d40bdcddc538290150d14efc8 in 1290ms, sequenceid=129, compaction requested=false 2024-11-14T02:44:41,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:41,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T02:44:41,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:41,372 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T02:44:41,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:41,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:41,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:41,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:41,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:41,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:41,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/72106d00f331486181e526e80d03d9bd is 50, key is test_row_0/A:col10/1731552280000/Put/seqid=0 2024-11-14T02:44:41,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742070_1246 (size=12151) 2024-11-14T02:44:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T02:44:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T02:44:41,782 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/72106d00f331486181e526e80d03d9bd 2024-11-14T02:44:41,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/12e0bf8716bd452b8ce8d9a4e2f87c96 is 50, key is test_row_0/B:col10/1731552280000/Put/seqid=0 2024-11-14T02:44:41,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742071_1247 (size=12151) 2024-11-14T02:44:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:42,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:42,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552282177, max timestamp=1731552282187Num uniques 859; 2024-11-14T02:44:42,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552282177, max timestamp=1731552282187Num uniques 859; 2024-11-14T02:44:42,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552282177, max timestamp=1731552282187Num uniques 859; 2024-11-14T02:44:42,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:42,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:42,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:42,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552342186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:42,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:42,195 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/12e0bf8716bd452b8ce8d9a4e2f87c96 2024-11-14T02:44:42,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/77c310698ed14ef4b08d6327d9059509 is 50, key is test_row_0/C:col10/1731552280000/Put/seqid=0 2024-11-14T02:44:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T02:44:42,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742072_1248 (size=12151) 2024-11-14T02:44:42,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552342229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:42,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552342230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:42,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:42,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552342295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:42,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552342345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:42,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552342345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:42,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:42,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552342505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:42,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:42,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552342555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:42,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552342555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:42,610 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/77c310698ed14ef4b08d6327d9059509 2024-11-14T02:44:42,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/72106d00f331486181e526e80d03d9bd as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/72106d00f331486181e526e80d03d9bd 2024-11-14T02:44:42,618 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/72106d00f331486181e526e80d03d9bd, entries=150, sequenceid=143, filesize=11.9 K 2024-11-14T02:44:42,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/12e0bf8716bd452b8ce8d9a4e2f87c96 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/12e0bf8716bd452b8ce8d9a4e2f87c96 2024-11-14T02:44:42,621 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/12e0bf8716bd452b8ce8d9a4e2f87c96, entries=150, sequenceid=143, filesize=11.9 K 2024-11-14T02:44:42,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/77c310698ed14ef4b08d6327d9059509 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/77c310698ed14ef4b08d6327d9059509 2024-11-14T02:44:42,625 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/77c310698ed14ef4b08d6327d9059509, entries=150, sequenceid=143, filesize=11.9 K 2024-11-14T02:44:42,626 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 21da8e2d40bdcddc538290150d14efc8 in 1254ms, sequenceid=143, compaction requested=true 2024-11-14T02:44:42,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:42,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:42,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-14T02:44:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-14T02:44:42,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-14T02:44:42,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5600 sec 2024-11-14T02:44:42,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.5650 sec 2024-11-14T02:44:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:42,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-14T02:44:42,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:42,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:42,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:42,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:42,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:42,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:42,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/26dcc0358ec540d38ce61fd3093d6fd3 is 50, key is test_row_0/A:col10/1731552282197/Put/seqid=0 2024-11-14T02:44:42,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742073_1249 (size=14541) 2024-11-14T02:44:42,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552342863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552342865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552342865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:42,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552342975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:43,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552343185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T02:44:43,216 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:43,216 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-14T02:44:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T02:44:43,218 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:43,218 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:43,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:43,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/26dcc0358ec540d38ce61fd3093d6fd3 2024-11-14T02:44:43,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/ade6b1f42f574eb1aa454bbc1cd9faa2 is 50, key is test_row_0/B:col10/1731552282197/Put/seqid=0 2024-11-14T02:44:43,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742074_1250 (size=12151) 2024-11-14T02:44:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T02:44:43,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T02:44:43,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:43,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:43,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:43,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:43,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:43,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552343375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:43,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552343375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:43,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552343495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:43,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T02:44:43,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:43,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:43,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:43,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:43,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T02:44:43,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/ade6b1f42f574eb1aa454bbc1cd9faa2 2024-11-14T02:44:43,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/68a84d2bd329429f9251394c9c7d57a0 is 50, key is test_row_0/C:col10/1731552282197/Put/seqid=0 2024-11-14T02:44:43,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742075_1251 (size=12151) 2024-11-14T02:44:43,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T02:44:43,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:43,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:43,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:43,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:43,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:43,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T02:44:43,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:43,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:43,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:43,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:43,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T02:44:43,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T02:44:43,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:43,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:43,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:43,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:43,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:44,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552344005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:44,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/68a84d2bd329429f9251394c9c7d57a0 2024-11-14T02:44:44,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/26dcc0358ec540d38ce61fd3093d6fd3 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/26dcc0358ec540d38ce61fd3093d6fd3 2024-11-14T02:44:44,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/26dcc0358ec540d38ce61fd3093d6fd3, entries=200, sequenceid=179, filesize=14.2 K 2024-11-14T02:44:44,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/ade6b1f42f574eb1aa454bbc1cd9faa2 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ade6b1f42f574eb1aa454bbc1cd9faa2 2024-11-14T02:44:44,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ade6b1f42f574eb1aa454bbc1cd9faa2, entries=150, sequenceid=179, filesize=11.9 K 2024-11-14T02:44:44,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/68a84d2bd329429f9251394c9c7d57a0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/68a84d2bd329429f9251394c9c7d57a0 2024-11-14T02:44:44,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/68a84d2bd329429f9251394c9c7d57a0, entries=150, sequenceid=179, filesize=11.9 K 2024-11-14T02:44:44,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 21da8e2d40bdcddc538290150d14efc8 in 1252ms, sequenceid=179, compaction requested=true 2024-11-14T02:44:44,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:44,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:44,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:44,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:44,071 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:44:44,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:44,071 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:44:44,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:44,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:44,072 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48560 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:44:44,073 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/B is initiating minor compaction (all files) 2024-11-14T02:44:44,073 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/B in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:44,073 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/05a246e0cc1b45399b276550d0ae9286, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/4892a92fcd7f4020b7353cba0c2a0e55, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/12e0bf8716bd452b8ce8d9a4e2f87c96, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ade6b1f42f574eb1aa454bbc1cd9faa2] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=47.4 K 2024-11-14T02:44:44,073 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53290 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:44:44,073 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/A is initiating minor compaction (all files) 2024-11-14T02:44:44,073 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/A in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:44,073 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e170a5eb5fc44198864f52402ddd2bde, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/644651748b6a403bb049eaf289f5f29f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/72106d00f331486181e526e80d03d9bd, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/26dcc0358ec540d38ce61fd3093d6fd3] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=52.0 K 2024-11-14T02:44:44,074 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 05a246e0cc1b45399b276550d0ae9286, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731552278178 2024-11-14T02:44:44,074 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting e170a5eb5fc44198864f52402ddd2bde, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731552278178 2024-11-14T02:44:44,074 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 4892a92fcd7f4020b7353cba0c2a0e55, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731552279341 2024-11-14T02:44:44,074 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 644651748b6a403bb049eaf289f5f29f, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731552279341 2024-11-14T02:44:44,074 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 12e0bf8716bd452b8ce8d9a4e2f87c96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731552279989 2024-11-14T02:44:44,075 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72106d00f331486181e526e80d03d9bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731552279989 2024-11-14T02:44:44,075 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26dcc0358ec540d38ce61fd3093d6fd3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731552282197 2024-11-14T02:44:44,075 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting ade6b1f42f574eb1aa454bbc1cd9faa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731552282197 2024-11-14T02:44:44,086 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#A#compaction#209 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:44,086 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/f1d8b35202d849d8bc721f2386c756c3 is 50, key is test_row_0/A:col10/1731552282197/Put/seqid=0 2024-11-14T02:44:44,089 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#B#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:44,090 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/588763612e1c478581b10ca4376d8c62 is 50, key is test_row_0/B:col10/1731552282197/Put/seqid=0 2024-11-14T02:44:44,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742076_1252 (size=12493) 2024-11-14T02:44:44,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742077_1253 (size=12493) 2024-11-14T02:44:44,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T02:44:44,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:44,134 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:44:44,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:44,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:44,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:44,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:44,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:44,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:44,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/cf7692a3c0f24e6b8758f6aaaff6a267 is 50, key is test_row_0/A:col10/1731552282854/Put/seqid=0 2024-11-14T02:44:44,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742078_1254 (size=12151) 2024-11-14T02:44:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T02:44:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:44,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:44,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552284430, max timestamp=1731552284445Num uniques 859; 2024-11-14T02:44:44,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:44,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552284430, max timestamp=1731552284445Num uniques 859; 2024-11-14T02:44:44,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:44,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552284430, max timestamp=1731552284445Num uniques 859; 2024-11-14T02:44:44,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:44,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552344498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:44,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552344499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:44,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:44,507 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/f1d8b35202d849d8bc721f2386c756c3 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f1d8b35202d849d8bc721f2386c756c3 2024-11-14T02:44:44,512 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/A of 21da8e2d40bdcddc538290150d14efc8 into f1d8b35202d849d8bc721f2386c756c3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:44,512 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:44,512 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/A, priority=12, startTime=1731552284071; duration=0sec 2024-11-14T02:44:44,512 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:44,512 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:A 2024-11-14T02:44:44,512 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:44:44,513 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48560 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:44:44,513 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/C is initiating minor compaction (all files) 2024-11-14T02:44:44,513 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/C in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:44,513 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/5ee4b821bd9b4c8fa3d5d2d226482ce3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/02db01f26d6649499d3e9c7f4053fe3f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/77c310698ed14ef4b08d6327d9059509, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/68a84d2bd329429f9251394c9c7d57a0] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=47.4 K 2024-11-14T02:44:44,513 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ee4b821bd9b4c8fa3d5d2d226482ce3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731552278178 2024-11-14T02:44:44,514 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02db01f26d6649499d3e9c7f4053fe3f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731552279341 2024-11-14T02:44:44,514 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77c310698ed14ef4b08d6327d9059509, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731552279989 2024-11-14T02:44:44,515 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68a84d2bd329429f9251394c9c7d57a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731552282197 2024-11-14T02:44:44,517 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/588763612e1c478581b10ca4376d8c62 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/588763612e1c478581b10ca4376d8c62 2024-11-14T02:44:44,521 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/B of 21da8e2d40bdcddc538290150d14efc8 into 588763612e1c478581b10ca4376d8c62(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:44,521 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:44,521 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/B, priority=12, startTime=1731552284071; duration=0sec 2024-11-14T02:44:44,522 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:44,522 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:B 2024-11-14T02:44:44,524 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:44,524 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/759e3c0aeff34939a8dcc84669f8bfe5 is 50, key is test_row_0/C:col10/1731552282197/Put/seqid=0 2024-11-14T02:44:44,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742079_1255 (size=12493) 2024-11-14T02:44:44,543 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/cf7692a3c0f24e6b8758f6aaaff6a267 2024-11-14T02:44:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/f2248093a18d4aa6a1fd7ff4789aa25e is 50, key is test_row_0/B:col10/1731552282854/Put/seqid=0 2024-11-14T02:44:44,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742080_1256 (size=12151) 2024-11-14T02:44:44,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1731552344615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:44,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552344616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:44,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:44,619 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in 21da8e2d40bdcddc538290150d14efc8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552344616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:44,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:44,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:44,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:44,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49758 deadline: 1731552344635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:44,640 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in 21da8e2d40bdcddc538290150d14efc8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:44,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552344825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:44,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552344826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:44,934 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/759e3c0aeff34939a8dcc84669f8bfe5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/759e3c0aeff34939a8dcc84669f8bfe5 2024-11-14T02:44:44,938 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/C of 21da8e2d40bdcddc538290150d14efc8 into 759e3c0aeff34939a8dcc84669f8bfe5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:44,938 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:44,938 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/C, priority=12, startTime=1731552284071; duration=0sec 2024-11-14T02:44:44,938 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:44,938 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:C 2024-11-14T02:44:44,956 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/f2248093a18d4aa6a1fd7ff4789aa25e 2024-11-14T02:44:44,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/da3ad4e1ac3f41ecb1c249d7f7f38c44 is 50, key is test_row_0/C:col10/1731552282854/Put/seqid=0 2024-11-14T02:44:44,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742081_1257 (size=12151) 2024-11-14T02:44:45,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:45,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552345015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:45,018 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_2' in 21da8e2d40bdcddc538290150d14efc8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:45,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552345135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:45,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:45,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552345135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T02:44:45,372 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/da3ad4e1ac3f41ecb1c249d7f7f38c44 2024-11-14T02:44:45,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/cf7692a3c0f24e6b8758f6aaaff6a267 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/cf7692a3c0f24e6b8758f6aaaff6a267 2024-11-14T02:44:45,387 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/cf7692a3c0f24e6b8758f6aaaff6a267, entries=150, sequenceid=189, filesize=11.9 K 2024-11-14T02:44:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/f2248093a18d4aa6a1fd7ff4789aa25e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/f2248093a18d4aa6a1fd7ff4789aa25e 2024-11-14T02:44:45,392 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/f2248093a18d4aa6a1fd7ff4789aa25e, entries=150, sequenceid=189, filesize=11.9 K 2024-11-14T02:44:45,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/da3ad4e1ac3f41ecb1c249d7f7f38c44 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/da3ad4e1ac3f41ecb1c249d7f7f38c44 2024-11-14T02:44:45,396 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/da3ad4e1ac3f41ecb1c249d7f7f38c44, entries=150, sequenceid=189, filesize=11.9 K 2024-11-14T02:44:45,397 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 21da8e2d40bdcddc538290150d14efc8 in 1263ms, sequenceid=189, compaction requested=false 2024-11-14T02:44:45,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:45,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:45,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-14T02:44:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-14T02:44:45,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-14T02:44:45,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1810 sec 2024-11-14T02:44:45,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.1850 sec 2024-11-14T02:44:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:45,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-14T02:44:45,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:45,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T02:44:45,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:45,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T02:44:45,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:45,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T02:44:45,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/1e08da3d12ec4bca904c41ed5b3285bd is 50, key is test_row_0/A:col10/1731552284491/Put/seqid=0 2024-11-14T02:44:45,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742082_1258 (size=16931) 2024-11-14T02:44:45,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552345662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:45,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552345667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:45,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552345775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:45,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552345776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:45,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552345985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:45,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552345985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:46,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/1e08da3d12ec4bca904c41ed5b3285bd 2024-11-14T02:44:46,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/dcbf32c85d934f22ba7fe41cd97cf1b1 is 50, key is test_row_0/B:col10/1731552284491/Put/seqid=0 2024-11-14T02:44:46,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742083_1259 (size=12151) 2024-11-14T02:44:46,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552346295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:46,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:46,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552346295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:46,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:46,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:46,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:46,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:46,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:46,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/dcbf32c85d934f22ba7fe41cd97cf1b1 2024-11-14T02:44:46,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/59aa934841a14cb6a1e7b1a6058073bf is 50, key is test_row_0/C:col10/1731552284491/Put/seqid=0 2024-11-14T02:44:46,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742084_1260 (size=12151) 2024-11-14T02:44:46,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552346805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:46,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552346805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:46,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/59aa934841a14cb6a1e7b1a6058073bf 2024-11-14T02:44:46,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/1e08da3d12ec4bca904c41ed5b3285bd as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1e08da3d12ec4bca904c41ed5b3285bd 2024-11-14T02:44:46,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1e08da3d12ec4bca904c41ed5b3285bd, entries=250, sequenceid=230, filesize=16.5 K 2024-11-14T02:44:46,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/dcbf32c85d934f22ba7fe41cd97cf1b1 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/dcbf32c85d934f22ba7fe41cd97cf1b1 2024-11-14T02:44:46,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/dcbf32c85d934f22ba7fe41cd97cf1b1, entries=150, sequenceid=230, filesize=11.9 K 2024-11-14T02:44:46,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/59aa934841a14cb6a1e7b1a6058073bf as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/59aa934841a14cb6a1e7b1a6058073bf 2024-11-14T02:44:46,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/59aa934841a14cb6a1e7b1a6058073bf, entries=150, sequenceid=230, filesize=11.9 K 2024-11-14T02:44:46,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 21da8e2d40bdcddc538290150d14efc8 in 1255ms, sequenceid=230, compaction requested=true 2024-11-14T02:44:46,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:46,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:46,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:46,905 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:46,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:46,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:46,905 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:46,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:46,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:46,906 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:46,906 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/B is initiating minor compaction (all files) 2024-11-14T02:44:46,906 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/B in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:46,906 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/588763612e1c478581b10ca4376d8c62, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/f2248093a18d4aa6a1fd7ff4789aa25e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/dcbf32c85d934f22ba7fe41cd97cf1b1] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=35.9 K 2024-11-14T02:44:46,906 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:46,906 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/A is initiating minor compaction (all files) 2024-11-14T02:44:46,906 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/A in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:46,906 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f1d8b35202d849d8bc721f2386c756c3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/cf7692a3c0f24e6b8758f6aaaff6a267, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1e08da3d12ec4bca904c41ed5b3285bd] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=40.6 K 2024-11-14T02:44:46,907 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 588763612e1c478581b10ca4376d8c62, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731552282197 2024-11-14T02:44:46,907 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1d8b35202d849d8bc721f2386c756c3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731552282197 2024-11-14T02:44:46,907 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting f2248093a18d4aa6a1fd7ff4789aa25e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731552282849 2024-11-14T02:44:46,907 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf7692a3c0f24e6b8758f6aaaff6a267, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731552282849 2024-11-14T02:44:46,907 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting dcbf32c85d934f22ba7fe41cd97cf1b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731552284491 2024-11-14T02:44:46,907 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e08da3d12ec4bca904c41ed5b3285bd, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731552284484 2024-11-14T02:44:46,914 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#A#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:46,914 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#B#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:46,914 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/1be5b490573b4caf8cb46a28fc7c55d0 is 50, key is test_row_0/A:col10/1731552284491/Put/seqid=0 2024-11-14T02:44:46,915 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/68ee76ed7576492e8915af48041b10a8 is 50, key is test_row_0/B:col10/1731552284491/Put/seqid=0 2024-11-14T02:44:46,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742085_1261 (size=12595) 2024-11-14T02:44:46,929 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/1be5b490573b4caf8cb46a28fc7c55d0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1be5b490573b4caf8cb46a28fc7c55d0 2024-11-14T02:44:46,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742086_1262 (size=12595) 2024-11-14T02:44:46,935 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/A of 21da8e2d40bdcddc538290150d14efc8 into 1be5b490573b4caf8cb46a28fc7c55d0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:46,935 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:46,935 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/A, priority=13, startTime=1731552286905; duration=0sec 2024-11-14T02:44:46,935 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:46,935 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:A 2024-11-14T02:44:46,935 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:46,936 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:46,936 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/C is initiating minor compaction (all files) 2024-11-14T02:44:46,936 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/C in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:46,936 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/759e3c0aeff34939a8dcc84669f8bfe5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/da3ad4e1ac3f41ecb1c249d7f7f38c44, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/59aa934841a14cb6a1e7b1a6058073bf] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=35.9 K 2024-11-14T02:44:46,937 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 759e3c0aeff34939a8dcc84669f8bfe5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731552282197 2024-11-14T02:44:46,937 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting da3ad4e1ac3f41ecb1c249d7f7f38c44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731552282849 2024-11-14T02:44:46,938 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59aa934841a14cb6a1e7b1a6058073bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731552284491 2024-11-14T02:44:46,943 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/68ee76ed7576492e8915af48041b10a8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/68ee76ed7576492e8915af48041b10a8 2024-11-14T02:44:46,952 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/B of 21da8e2d40bdcddc538290150d14efc8 into 68ee76ed7576492e8915af48041b10a8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:46,952 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:46,952 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/B, priority=13, startTime=1731552286905; duration=0sec 2024-11-14T02:44:46,952 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:46,952 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:B 2024-11-14T02:44:46,954 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#C#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:46,955 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/b9b0f3a9165046a696a0f9525e07f4a2 is 50, key is test_row_0/C:col10/1731552284491/Put/seqid=0 2024-11-14T02:44:46,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742087_1263 (size=12595) 2024-11-14T02:44:46,970 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/b9b0f3a9165046a696a0f9525e07f4a2 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/b9b0f3a9165046a696a0f9525e07f4a2 2024-11-14T02:44:46,975 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/C of 21da8e2d40bdcddc538290150d14efc8 into b9b0f3a9165046a696a0f9525e07f4a2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:46,975 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:46,975 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/C, priority=13, startTime=1731552286905; duration=0sec 2024-11-14T02:44:46,975 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:46,976 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:C 2024-11-14T02:44:47,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:47,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:44:47,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:47,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:47,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:47,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:47,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:47,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:47,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/63f265c5cf86436a8d2bdcf1f811f99c is 50, key is test_row_0/A:col10/1731552287037/Put/seqid=0 2024-11-14T02:44:47,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742088_1264 (size=12151) 2024-11-14T02:44:47,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/63f265c5cf86436a8d2bdcf1f811f99c 2024-11-14T02:44:47,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/ab18cae7292f470d911721ca1888500b is 50, key is test_row_0/B:col10/1731552287037/Put/seqid=0 2024-11-14T02:44:47,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742089_1265 (size=12151) 2024-11-14T02:44:47,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/ab18cae7292f470d911721ca1888500b 2024-11-14T02:44:47,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/3315783654904ed284870f15573491dc is 50, key is test_row_0/C:col10/1731552287037/Put/seqid=0 2024-11-14T02:44:47,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742090_1266 (size=12151) 2024-11-14T02:44:47,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552287086, max timestamp=1731552287103Num uniques 859; 2024-11-14T02:44:47,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552287086, max timestamp=1731552287103Num uniques 859; 2024-11-14T02:44:47,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:44:47,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:44:47,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552287086, max timestamp=1731552287103Num uniques 859; 2024-11-14T02:44:47,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:44:47,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552347185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:47,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552347295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:47,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T02:44:47,375 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:47,376 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:47,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-14T02:44:47,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T02:44:47,377 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:47,378 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:47,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T02:44:47,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/3315783654904ed284870f15573491dc 2024-11-14T02:44:47,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/63f265c5cf86436a8d2bdcf1f811f99c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/63f265c5cf86436a8d2bdcf1f811f99c 2024-11-14T02:44:47,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/63f265c5cf86436a8d2bdcf1f811f99c, entries=150, sequenceid=243, filesize=11.9 K 2024-11-14T02:44:47,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/ab18cae7292f470d911721ca1888500b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ab18cae7292f470d911721ca1888500b 2024-11-14T02:44:47,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ab18cae7292f470d911721ca1888500b, entries=150, sequenceid=243, filesize=11.9 K 2024-11-14T02:44:47,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/3315783654904ed284870f15573491dc as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3315783654904ed284870f15573491dc 2024-11-14T02:44:47,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3315783654904ed284870f15573491dc, entries=150, sequenceid=243, filesize=11.9 K 2024-11-14T02:44:47,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 21da8e2d40bdcddc538290150d14efc8 in 466ms, sequenceid=243, compaction requested=false 2024-11-14T02:44:47,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:47,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-14T02:44:47,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:47,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T02:44:47,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:47,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T02:44:47,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:47,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T02:44:47,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/042a39d8c5a747a4bbb005daed7489b0 is 50, key is test_row_0/A:col10/1731552287148/Put/seqid=0 2024-11-14T02:44:47,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742091_1267 (size=14741) 2024-11-14T02:44:47,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/042a39d8c5a747a4bbb005daed7489b0 2024-11-14T02:44:47,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T02:44:47,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:47,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:47,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:47,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/c5c66e6c879f46d6aa58dd6b472a1faa is 50, key is test_row_0/B:col10/1731552287148/Put/seqid=0 2024-11-14T02:44:47,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552347539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:47,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:47,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742092_1268 (size=12301) 2024-11-14T02:44:47,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552347655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:47,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T02:44:47,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:47,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:47,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:47,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T02:44:47,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552347815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:47,818 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_2' in 21da8e2d40bdcddc538290150d14efc8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:47,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:47,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552347825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:47,826 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_0' in 21da8e2d40bdcddc538290150d14efc8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:47,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T02:44:47,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:47,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:47,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:47,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552347865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:47,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/c5c66e6c879f46d6aa58dd6b472a1faa 2024-11-14T02:44:47,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/b56c609b68ba44b482bfe882067d10fc is 50, key is test_row_0/C:col10/1731552287148/Put/seqid=0 2024-11-14T02:44:47,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742093_1269 (size=12301) 2024-11-14T02:44:47,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T02:44:47,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:47,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:47,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:47,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T02:44:48,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T02:44:48,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:48,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:48,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:48,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552348175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:48,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:48,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T02:44:48,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:48,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:48,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:48,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:48,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/b56c609b68ba44b482bfe882067d10fc 2024-11-14T02:44:48,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/042a39d8c5a747a4bbb005daed7489b0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/042a39d8c5a747a4bbb005daed7489b0 2024-11-14T02:44:48,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/042a39d8c5a747a4bbb005daed7489b0, entries=200, sequenceid=281, filesize=14.4 K 2024-11-14T02:44:48,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/c5c66e6c879f46d6aa58dd6b472a1faa as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c5c66e6c879f46d6aa58dd6b472a1faa 2024-11-14T02:44:48,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c5c66e6c879f46d6aa58dd6b472a1faa, entries=150, sequenceid=281, filesize=12.0 K 2024-11-14T02:44:48,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/b56c609b68ba44b482bfe882067d10fc as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/b56c609b68ba44b482bfe882067d10fc 2024-11-14T02:44:48,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/b56c609b68ba44b482bfe882067d10fc, entries=150, sequenceid=281, filesize=12.0 K 2024-11-14T02:44:48,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 21da8e2d40bdcddc538290150d14efc8 in 874ms, sequenceid=281, compaction requested=true 2024-11-14T02:44:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:48,381 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T02:44:48,381 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:48,383 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39487 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:48,384 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:48,384 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/A is initiating minor compaction (all files) 2024-11-14T02:44:48,384 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/B is initiating minor compaction (all files) 2024-11-14T02:44:48,384 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/B in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:48,384 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/A in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:48,384 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/68ee76ed7576492e8915af48041b10a8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ab18cae7292f470d911721ca1888500b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c5c66e6c879f46d6aa58dd6b472a1faa] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=36.2 K 2024-11-14T02:44:48,384 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1be5b490573b4caf8cb46a28fc7c55d0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/63f265c5cf86436a8d2bdcf1f811f99c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/042a39d8c5a747a4bbb005daed7489b0] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=38.6 K 2024-11-14T02:44:48,385 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1be5b490573b4caf8cb46a28fc7c55d0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731552284491 2024-11-14T02:44:48,385 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 68ee76ed7576492e8915af48041b10a8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731552284491 2024-11-14T02:44:48,385 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63f265c5cf86436a8d2bdcf1f811f99c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731552285666 2024-11-14T02:44:48,385 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting ab18cae7292f470d911721ca1888500b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731552285666 2024-11-14T02:44:48,386 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 042a39d8c5a747a4bbb005daed7489b0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731552287148 2024-11-14T02:44:48,386 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting c5c66e6c879f46d6aa58dd6b472a1faa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731552287148 2024-11-14T02:44:48,392 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#B#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:48,392 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#A#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:48,393 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/73ef4b9b60cc43d1900cf77baece76d2 is 50, key is test_row_0/A:col10/1731552287148/Put/seqid=0 2024-11-14T02:44:48,393 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/7d71a58e013f454896f3c49cb110f658 is 50, key is test_row_0/B:col10/1731552287148/Put/seqid=0 2024-11-14T02:44:48,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742094_1270 (size=12847) 2024-11-14T02:44:48,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742095_1271 (size=12847) 2024-11-14T02:44:48,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T02:44:48,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:48,447 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-14T02:44:48,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:48,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:48,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:48,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:48,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:48,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:48,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/5c2f0ad78c6d47ecb31d2163dbdf76e5 is 50, key is test_row_0/A:col10/1731552287516/Put/seqid=0 2024-11-14T02:44:48,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742096_1272 (size=12301) 2024-11-14T02:44:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T02:44:48,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:48,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552288745, max timestamp=1731552288760Num uniques 859; 2024-11-14T02:44:48,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552288745, max timestamp=1731552288760Num uniques 859; 2024-11-14T02:44:48,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552288745, max timestamp=1731552288760Num uniques 859; 2024-11-14T02:44:48,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:44:48,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:44:48,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:44:48,801 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/7d71a58e013f454896f3c49cb110f658 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/7d71a58e013f454896f3c49cb110f658 2024-11-14T02:44:48,806 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/73ef4b9b60cc43d1900cf77baece76d2 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/73ef4b9b60cc43d1900cf77baece76d2 2024-11-14T02:44:48,806 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/B of 21da8e2d40bdcddc538290150d14efc8 into 7d71a58e013f454896f3c49cb110f658(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:48,806 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:48,806 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/B, priority=13, startTime=1731552288381; duration=0sec 2024-11-14T02:44:48,806 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:48,806 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:B 2024-11-14T02:44:48,806 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:48,808 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:48,808 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/C is initiating minor compaction (all files) 2024-11-14T02:44:48,808 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/C in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:48,808 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/b9b0f3a9165046a696a0f9525e07f4a2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3315783654904ed284870f15573491dc, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/b56c609b68ba44b482bfe882067d10fc] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=36.2 K 2024-11-14T02:44:48,808 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting b9b0f3a9165046a696a0f9525e07f4a2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731552284491 2024-11-14T02:44:48,809 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 3315783654904ed284870f15573491dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731552285666 2024-11-14T02:44:48,809 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting b56c609b68ba44b482bfe882067d10fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731552287148 2024-11-14T02:44:48,810 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/A of 21da8e2d40bdcddc538290150d14efc8 into 73ef4b9b60cc43d1900cf77baece76d2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:48,810 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:48,810 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/A, priority=13, startTime=1731552288381; duration=0sec 2024-11-14T02:44:48,810 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:48,810 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:A 2024-11-14T02:44:48,819 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#C#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:48,820 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/6655f3f164f64960be22a406a1a3d1e7 is 50, key is test_row_0/C:col10/1731552287148/Put/seqid=0 2024-11-14T02:44:48,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742097_1273 (size=12847) 2024-11-14T02:44:48,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731552288827, max timestamp=1731552288850Num uniques 1018; 2024-11-14T02:44:48,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T02:44:48,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731552288827, max timestamp=1731552288850Num uniques 1018; 2024-11-14T02:44:48,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731552288827, max timestamp=1731552288850Num uniques 1018; 2024-11-14T02:44:48,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T02:44:48,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T02:44:48,856 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/5c2f0ad78c6d47ecb31d2163dbdf76e5 2024-11-14T02:44:48,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/be19b12086784830916be20a7eb82caf is 50, key is test_row_0/B:col10/1731552287516/Put/seqid=0 2024-11-14T02:44:48,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742098_1274 (size=12301) 2024-11-14T02:44:48,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731552288921, max timestamp=1731552288941Num uniques 1027; 2024-11-14T02:44:48,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-14T02:44:48,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731552288921, max timestamp=1731552288941Num uniques 1027; 2024-11-14T02:44:48,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-14T02:44:48,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731552288921, max timestamp=1731552288941Num uniques 1027; 2024-11-14T02:44:48,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-14T02:44:49,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731552288991, max timestamp=1731552289029Num uniques 1036; 2024-11-14T02:44:49,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731552288991, max timestamp=1731552289029Num uniques 1036; 2024-11-14T02:44:49,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731552288991, max timestamp=1731552289029Num uniques 1036; 2024-11-14T02:44:49,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-14T02:44:49,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-14T02:44:49,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-14T02:44:49,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731552289075, max timestamp=1731552289111Num uniques 1045; 2024-11-14T02:44:49,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T02:44:49,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731552289075, max timestamp=1731552289111Num uniques 1045; 2024-11-14T02:44:49,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T02:44:49,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731552289075, max timestamp=1731552289111Num uniques 1045; 2024-11-14T02:44:49,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T02:44:49,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552349204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:49,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:49,229 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/6655f3f164f64960be22a406a1a3d1e7 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6655f3f164f64960be22a406a1a3d1e7 2024-11-14T02:44:49,234 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/C of 21da8e2d40bdcddc538290150d14efc8 into 6655f3f164f64960be22a406a1a3d1e7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:49,234 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:49,234 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/C, priority=13, startTime=1731552288381; duration=0sec 2024-11-14T02:44:49,234 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:49,234 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:C 2024-11-14T02:44:49,270 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/be19b12086784830916be20a7eb82caf 2024-11-14T02:44:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/85f2a88fbb2042818438d8fa4224f58a is 50, key is test_row_0/C:col10/1731552287516/Put/seqid=0 2024-11-14T02:44:49,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742099_1275 (size=12301) 2024-11-14T02:44:49,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:49,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552349316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T02:44:49,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552349525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:49,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:49,680 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/85f2a88fbb2042818438d8fa4224f58a 2024-11-14T02:44:49,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/5c2f0ad78c6d47ecb31d2163dbdf76e5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/5c2f0ad78c6d47ecb31d2163dbdf76e5 2024-11-14T02:44:49,688 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/5c2f0ad78c6d47ecb31d2163dbdf76e5, entries=150, sequenceid=290, filesize=12.0 K 2024-11-14T02:44:49,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/be19b12086784830916be20a7eb82caf as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/be19b12086784830916be20a7eb82caf 2024-11-14T02:44:49,692 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/be19b12086784830916be20a7eb82caf, entries=150, sequenceid=290, filesize=12.0 K 2024-11-14T02:44:49,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/85f2a88fbb2042818438d8fa4224f58a as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/85f2a88fbb2042818438d8fa4224f58a 2024-11-14T02:44:49,696 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/85f2a88fbb2042818438d8fa4224f58a, entries=150, sequenceid=290, filesize=12.0 K 2024-11-14T02:44:49,697 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 21da8e2d40bdcddc538290150d14efc8 in 1250ms, sequenceid=290, compaction requested=false 2024-11-14T02:44:49,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:49,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:49,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-14T02:44:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-14T02:44:49,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-14T02:44:49,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3200 sec 2024-11-14T02:44:49,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.3240 sec 2024-11-14T02:44:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:49,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=147.60 KB heapSize=422.16 KB 2024-11-14T02:44:49,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:49,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T02:44:49,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:49,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T02:44:49,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:49,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T02:44:49,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/4ad8aacd03e34a638b64c181b2e29e6c is 50, key is test_row_0/A:col10/1731552289179/Put/seqid=0 2024-11-14T02:44:49,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552349847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:49,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552349848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552349848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:49,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742100_1276 (size=14741) 2024-11-14T02:44:49,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:49,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552349955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552349955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:49,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552349955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:50,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552350165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552350165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:50,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552350165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:50,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/4ad8aacd03e34a638b64c181b2e29e6c 2024-11-14T02:44:50,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/13820fb862ae458287a490bba814cd5c is 50, key is test_row_0/B:col10/1731552289179/Put/seqid=0 2024-11-14T02:44:50,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742101_1277 (size=12301) 2024-11-14T02:44:50,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:50,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552350475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552350475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:50,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552350475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:50,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/13820fb862ae458287a490bba814cd5c 2024-11-14T02:44:50,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/6ccac307e997448e90f13787b83f8348 is 50, key is test_row_0/C:col10/1731552289179/Put/seqid=0 2024-11-14T02:44:50,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742102_1278 (size=12301) 2024-11-14T02:44:50,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/6ccac307e997448e90f13787b83f8348 2024-11-14T02:44:50,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/4ad8aacd03e34a638b64c181b2e29e6c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/4ad8aacd03e34a638b64c181b2e29e6c 2024-11-14T02:44:50,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/4ad8aacd03e34a638b64c181b2e29e6c, entries=200, sequenceid=400, filesize=14.4 K 2024-11-14T02:44:50,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/13820fb862ae458287a490bba814cd5c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/13820fb862ae458287a490bba814cd5c 2024-11-14T02:44:50,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/13820fb862ae458287a490bba814cd5c, entries=150, sequenceid=400, filesize=12.0 K 2024-11-14T02:44:50,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/6ccac307e997448e90f13787b83f8348 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6ccac307e997448e90f13787b83f8348 2024-11-14T02:44:50,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6ccac307e997448e90f13787b83f8348, entries=150, sequenceid=400, filesize=12.0 K 2024-11-14T02:44:50,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~422.11 KB/432240, currentSize=33.54 KB/34350 for 21da8e2d40bdcddc538290150d14efc8 in 871ms, sequenceid=400, compaction requested=true 2024-11-14T02:44:50,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:50,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:50,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:50,709 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:50,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:50,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:50,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:50,709 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:50,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:50,710 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:50,710 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:50,710 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/B is initiating minor compaction (all files) 2024-11-14T02:44:50,710 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/A is initiating minor compaction (all files) 2024-11-14T02:44:50,710 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/A in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:50,710 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/B in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:50,710 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/73ef4b9b60cc43d1900cf77baece76d2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/5c2f0ad78c6d47ecb31d2163dbdf76e5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/4ad8aacd03e34a638b64c181b2e29e6c] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=39.0 K 2024-11-14T02:44:50,710 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/7d71a58e013f454896f3c49cb110f658, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/be19b12086784830916be20a7eb82caf, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/13820fb862ae458287a490bba814cd5c] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=36.6 K 2024-11-14T02:44:50,710 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d71a58e013f454896f3c49cb110f658, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731552287148 2024-11-14T02:44:50,710 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73ef4b9b60cc43d1900cf77baece76d2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731552287148 2024-11-14T02:44:50,711 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c2f0ad78c6d47ecb31d2163dbdf76e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731552287516 2024-11-14T02:44:50,711 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting be19b12086784830916be20a7eb82caf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731552287516 2024-11-14T02:44:50,711 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ad8aacd03e34a638b64c181b2e29e6c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731552289179 2024-11-14T02:44:50,711 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 13820fb862ae458287a490bba814cd5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731552289179 2024-11-14T02:44:50,721 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#A#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:50,721 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/6bdc30d9ed244d3aa07ba8098b3e8b19 is 50, key is test_row_0/A:col10/1731552289179/Put/seqid=0 2024-11-14T02:44:50,722 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#B#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:50,722 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/b3e856066f95497ebf1300dc975969d7 is 50, key is test_row_0/B:col10/1731552289179/Put/seqid=0 2024-11-14T02:44:50,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742103_1279 (size=12949) 2024-11-14T02:44:50,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742104_1280 (size=12949) 2024-11-14T02:44:50,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:50,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:44:50,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:50,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:50,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:50,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:50,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:50,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:50,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/f3ab64084d914f79ae3139ccf3b3bad3 is 50, key is test_row_0/A:col10/1731552290986/Put/seqid=0 2024-11-14T02:44:51,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742105_1281 (size=14741) 2024-11-14T02:44:51,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/f3ab64084d914f79ae3139ccf3b3bad3 2024-11-14T02:44:51,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/2c686021e5c24d8499dae37efa18efea is 50, key is test_row_0/B:col10/1731552290986/Put/seqid=0 2024-11-14T02:44:51,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552351024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:51,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552291012, max timestamp=1731552291026Num uniques 859; 2024-11-14T02:44:51,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552291012, max timestamp=1731552291026Num uniques 859; 2024-11-14T02:44:51,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552291012, max timestamp=1731552291026Num uniques 859; 2024-11-14T02:44:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:51,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:51,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742106_1282 (size=12301) 2024-11-14T02:44:51,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552351087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:51,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552351088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:51,136 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/6bdc30d9ed244d3aa07ba8098b3e8b19 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/6bdc30d9ed244d3aa07ba8098b3e8b19 2024-11-14T02:44:51,136 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/b3e856066f95497ebf1300dc975969d7 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/b3e856066f95497ebf1300dc975969d7 2024-11-14T02:44:51,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552351135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:51,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:51,141 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/B of 21da8e2d40bdcddc538290150d14efc8 into b3e856066f95497ebf1300dc975969d7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:51,141 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:51,141 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/A of 21da8e2d40bdcddc538290150d14efc8 into 6bdc30d9ed244d3aa07ba8098b3e8b19(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:51,141 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:51,141 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/B, priority=13, startTime=1731552290709; duration=0sec 2024-11-14T02:44:51,141 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/A, priority=13, startTime=1731552290709; duration=0sec 2024-11-14T02:44:51,141 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:51,141 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:B 2024-11-14T02:44:51,141 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:51,141 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:A 2024-11-14T02:44:51,141 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:51,142 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:51,142 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/C is initiating minor compaction (all files) 2024-11-14T02:44:51,142 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/C in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:51,142 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6655f3f164f64960be22a406a1a3d1e7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/85f2a88fbb2042818438d8fa4224f58a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6ccac307e997448e90f13787b83f8348] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=36.6 K 2024-11-14T02:44:51,143 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 6655f3f164f64960be22a406a1a3d1e7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731552287148 2024-11-14T02:44:51,143 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 85f2a88fbb2042818438d8fa4224f58a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731552287516 2024-11-14T02:44:51,143 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ccac307e997448e90f13787b83f8348, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731552289179 2024-11-14T02:44:51,149 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#C#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:51,150 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/eb3dcba7e91d40fe8880bdb1c2703ed4 is 50, key is test_row_0/C:col10/1731552289179/Put/seqid=0 2024-11-14T02:44:51,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742107_1283 (size=12949) 2024-11-14T02:44:51,157 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/eb3dcba7e91d40fe8880bdb1c2703ed4 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/eb3dcba7e91d40fe8880bdb1c2703ed4 2024-11-14T02:44:51,164 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/C of 21da8e2d40bdcddc538290150d14efc8 into eb3dcba7e91d40fe8880bdb1c2703ed4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:51,164 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:51,164 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/C, priority=13, startTime=1731552290709; duration=0sec 2024-11-14T02:44:51,164 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:51,164 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:C 2024-11-14T02:44:51,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552351195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:51,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:51,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552351195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:51,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:51,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:51,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552351345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:51,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552351405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552351405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:51,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/2c686021e5c24d8499dae37efa18efea 2024-11-14T02:44:51,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/74260c3f6c5a4ccbb6206cd0ad97e956 is 50, key is test_row_0/C:col10/1731552290986/Put/seqid=0 2024-11-14T02:44:51,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742108_1284 (size=12301) 2024-11-14T02:44:51,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/74260c3f6c5a4ccbb6206cd0ad97e956 2024-11-14T02:44:51,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/f3ab64084d914f79ae3139ccf3b3bad3 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f3ab64084d914f79ae3139ccf3b3bad3 2024-11-14T02:44:51,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f3ab64084d914f79ae3139ccf3b3bad3, entries=200, sequenceid=411, filesize=14.4 K 2024-11-14T02:44:51,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/2c686021e5c24d8499dae37efa18efea as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/2c686021e5c24d8499dae37efa18efea 2024-11-14T02:44:51,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/2c686021e5c24d8499dae37efa18efea, entries=150, sequenceid=411, filesize=12.0 K 2024-11-14T02:44:51,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/74260c3f6c5a4ccbb6206cd0ad97e956 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/74260c3f6c5a4ccbb6206cd0ad97e956 2024-11-14T02:44:51,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/74260c3f6c5a4ccbb6206cd0ad97e956, entries=150, sequenceid=411, filesize=12.0 K 2024-11-14T02:44:51,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 21da8e2d40bdcddc538290150d14efc8 in 480ms, sequenceid=411, compaction requested=false 2024-11-14T02:44:51,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T02:44:51,535 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:51,536 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-14T02:44:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T02:44:51,537 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:51,538 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:51,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T02:44:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:51,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-14T02:44:51,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:51,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:51,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:51,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:51,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:51,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:51,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/1ef290d2c00d4b58889b6c754163b50c is 50, key is test_row_0/A:col10/1731552291658/Put/seqid=0 2024-11-14T02:44:51,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742109_1285 (size=14741) 2024-11-14T02:44:51,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:51,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:51,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:51,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552351698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:51,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:51,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:51,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552351715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552351714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:51,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552351805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:51,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:51,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:51,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:51,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:51,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:51,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T02:44:51,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:51,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:52,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552352015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:52,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/1ef290d2c00d4b58889b6c754163b50c 2024-11-14T02:44:52,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/ca3695839d8440f898d37cd679cdc253 is 50, key is test_row_0/B:col10/1731552291658/Put/seqid=0 2024-11-14T02:44:52,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742110_1286 (size=12301) 2024-11-14T02:44:52,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:52,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:52,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:52,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:52,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T02:44:52,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552352225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:52,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552352235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:52,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:52,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:52,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:52,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:52,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552352325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:52,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:52,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:52,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:52,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:52,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:52,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/ca3695839d8440f898d37cd679cdc253 2024-11-14T02:44:52,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/c36f2c9b101f45178b9d2656e219c826 is 50, key is test_row_0/C:col10/1731552291658/Put/seqid=0 2024-11-14T02:44:52,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742111_1287 (size=12301) 2024-11-14T02:44:52,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:52,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:52,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:52,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:52,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T02:44:52,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:52,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:52,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:52,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:52,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552352835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:52,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/c36f2c9b101f45178b9d2656e219c826 2024-11-14T02:44:52,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/1ef290d2c00d4b58889b6c754163b50c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1ef290d2c00d4b58889b6c754163b50c 2024-11-14T02:44:52,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1ef290d2c00d4b58889b6c754163b50c, entries=200, sequenceid=450, filesize=14.4 K 2024-11-14T02:44:52,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/ca3695839d8440f898d37cd679cdc253 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ca3695839d8440f898d37cd679cdc253 2024-11-14T02:44:52,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ca3695839d8440f898d37cd679cdc253, entries=150, sequenceid=450, filesize=12.0 K 2024-11-14T02:44:52,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/c36f2c9b101f45178b9d2656e219c826 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/c36f2c9b101f45178b9d2656e219c826 2024-11-14T02:44:52,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/c36f2c9b101f45178b9d2656e219c826, entries=150, sequenceid=450, filesize=12.0 K 2024-11-14T02:44:52,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 21da8e2d40bdcddc538290150d14efc8 in 1253ms, sequenceid=450, compaction requested=true 2024-11-14T02:44:52,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:52,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:52,911 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:52,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:52,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:52,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:52,911 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:52,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:52,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:52,912 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:52,912 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:52,912 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/A is initiating minor compaction (all files) 2024-11-14T02:44:52,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T02:44:52,912 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/B is initiating minor compaction (all files) 2024-11-14T02:44:52,912 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/A in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:52,912 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/B in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:52,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:52,912 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/6bdc30d9ed244d3aa07ba8098b3e8b19, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f3ab64084d914f79ae3139ccf3b3bad3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1ef290d2c00d4b58889b6c754163b50c] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=41.4 K 2024-11-14T02:44:52,912 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/b3e856066f95497ebf1300dc975969d7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/2c686021e5c24d8499dae37efa18efea, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ca3695839d8440f898d37cd679cdc253] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=36.7 K 2024-11-14T02:44:52,912 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:44:52,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:52,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:52,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:52,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:52,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:52,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:52,913 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting b3e856066f95497ebf1300dc975969d7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731552289179 2024-11-14T02:44:52,913 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bdc30d9ed244d3aa07ba8098b3e8b19, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731552289179 2024-11-14T02:44:52,914 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c686021e5c24d8499dae37efa18efea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731552290986 2024-11-14T02:44:52,915 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3ab64084d914f79ae3139ccf3b3bad3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731552289847 2024-11-14T02:44:52,915 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting ca3695839d8440f898d37cd679cdc253, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731552291079 2024-11-14T02:44:52,915 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ef290d2c00d4b58889b6c754163b50c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731552291079 2024-11-14T02:44:52,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/d2ce6d7adfbe40a38a16785905030ec9 is 50, key is test_row_0/A:col10/1731552291691/Put/seqid=0 2024-11-14T02:44:52,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742112_1288 (size=12301) 2024-11-14T02:44:52,926 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#B#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:52,926 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#A#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:52,926 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/bb497c2631ee49eca465c4745b8713da is 50, key is test_row_0/B:col10/1731552291658/Put/seqid=0 2024-11-14T02:44:52,926 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/e707a7eb9ed64487ae115983807db23e is 50, key is test_row_0/A:col10/1731552291658/Put/seqid=0 2024-11-14T02:44:52,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742113_1289 (size=13051) 2024-11-14T02:44:52,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742114_1290 (size=13051) 2024-11-14T02:44:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:53,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:53,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552293281, max timestamp=1731552293291Num uniques 859; 2024-11-14T02:44:53,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:53,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552293281, max timestamp=1731552293291Num uniques 859; 2024-11-14T02:44:53,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:53,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552293281, max timestamp=1731552293291Num uniques 859; 2024-11-14T02:44:53,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:53,323 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/d2ce6d7adfbe40a38a16785905030ec9 2024-11-14T02:44:53,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/c76b659570314f6a87d9d677ffc62272 is 50, key is test_row_0/B:col10/1731552291691/Put/seqid=0 2024-11-14T02:44:53,337 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/bb497c2631ee49eca465c4745b8713da as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/bb497c2631ee49eca465c4745b8713da 2024-11-14T02:44:53,338 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/e707a7eb9ed64487ae115983807db23e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e707a7eb9ed64487ae115983807db23e 2024-11-14T02:44:53,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552353334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:53,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552353336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:53,343 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/B of 21da8e2d40bdcddc538290150d14efc8 into bb497c2631ee49eca465c4745b8713da(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:53,343 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:53,343 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/B, priority=13, startTime=1731552292911; duration=0sec 2024-11-14T02:44:53,343 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:53,343 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:B 2024-11-14T02:44:53,343 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:53,344 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:53,344 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/C is initiating minor compaction (all files) 2024-11-14T02:44:53,344 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/C in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:53,344 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/eb3dcba7e91d40fe8880bdb1c2703ed4, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/74260c3f6c5a4ccbb6206cd0ad97e956, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/c36f2c9b101f45178b9d2656e219c826] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=36.7 K 2024-11-14T02:44:53,345 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/A of 21da8e2d40bdcddc538290150d14efc8 into e707a7eb9ed64487ae115983807db23e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:53,345 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:53,345 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/A, priority=13, startTime=1731552292911; duration=0sec 2024-11-14T02:44:53,345 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:53,345 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:A 2024-11-14T02:44:53,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742115_1291 (size=12301) 2024-11-14T02:44:53,345 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting eb3dcba7e91d40fe8880bdb1c2703ed4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731552289179 2024-11-14T02:44:53,345 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/c76b659570314f6a87d9d677ffc62272 2024-11-14T02:44:53,346 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 74260c3f6c5a4ccbb6206cd0ad97e956, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731552290986 2024-11-14T02:44:53,346 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting c36f2c9b101f45178b9d2656e219c826, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731552291079 2024-11-14T02:44:53,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/65289f31143143959954820e7190cd81 is 50, key is test_row_0/C:col10/1731552291691/Put/seqid=0 2024-11-14T02:44:53,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742116_1292 (size=12301) 2024-11-14T02:44:53,357 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#C#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:53,358 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/65289f31143143959954820e7190cd81 2024-11-14T02:44:53,358 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/d393a8dde18b4ceba995b94450179d39 is 50, key is test_row_0/C:col10/1731552291658/Put/seqid=0 2024-11-14T02:44:53,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742117_1293 (size=13051) 2024-11-14T02:44:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/d2ce6d7adfbe40a38a16785905030ec9 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/d2ce6d7adfbe40a38a16785905030ec9 2024-11-14T02:44:53,366 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/d2ce6d7adfbe40a38a16785905030ec9, entries=150, sequenceid=460, filesize=12.0 K 2024-11-14T02:44:53,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/c76b659570314f6a87d9d677ffc62272 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c76b659570314f6a87d9d677ffc62272 2024-11-14T02:44:53,371 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c76b659570314f6a87d9d677ffc62272, entries=150, sequenceid=460, filesize=12.0 K 2024-11-14T02:44:53,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/65289f31143143959954820e7190cd81 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/65289f31143143959954820e7190cd81 2024-11-14T02:44:53,375 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/65289f31143143959954820e7190cd81, entries=150, sequenceid=460, filesize=12.0 K 2024-11-14T02:44:53,376 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 21da8e2d40bdcddc538290150d14efc8 in 464ms, sequenceid=460, compaction requested=false 2024-11-14T02:44:53,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:53,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:53,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-14T02:44:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-14T02:44:53,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-14T02:44:53,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8400 sec 2024-11-14T02:44:53,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.8440 sec 2024-11-14T02:44:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:53,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-14T02:44:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:53,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/79c1527f1d9f4ce5a546911ad6cb9877 is 50, key is test_row_0/A:col10/1731552293448/Put/seqid=0 2024-11-14T02:44:53,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742118_1294 (size=14741) 2024-11-14T02:44:53,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552353461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:53,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:53,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552353466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:53,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552353575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552353575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T02:44:53,685 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:53,685 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:44:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-14T02:44:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T02:44:53,687 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:44:53,688 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:44:53,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:44:53,767 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/d393a8dde18b4ceba995b94450179d39 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/d393a8dde18b4ceba995b94450179d39 2024-11-14T02:44:53,772 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/C of 21da8e2d40bdcddc538290150d14efc8 into d393a8dde18b4ceba995b94450179d39(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:53,772 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:53,772 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/C, priority=13, startTime=1731552292911; duration=0sec 2024-11-14T02:44:53,772 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:53,772 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:C 2024-11-14T02:44:53,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552353785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:53,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552353785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T02:44:53,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T02:44:53,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:53,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:53,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:53,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:53,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:53,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49766 deadline: 1731552353855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:53,858 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_2' in 21da8e2d40bdcddc538290150d14efc8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:53,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/79c1527f1d9f4ce5a546911ad6cb9877 2024-11-14T02:44:53,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/c2a9497cdd1b47b59b06cc65eccabcb5 is 50, key is test_row_0/B:col10/1731552293448/Put/seqid=0 2024-11-14T02:44:53,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742119_1295 (size=12301) 2024-11-14T02:44:53,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/c2a9497cdd1b47b59b06cc65eccabcb5 2024-11-14T02:44:53,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/0d00a06fe6744744ac5f681b6cc3c2e6 is 50, key is test_row_0/C:col10/1731552293448/Put/seqid=0 2024-11-14T02:44:53,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742120_1296 (size=12301) 2024-11-14T02:44:53,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T02:44:53,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:53,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:53,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:53,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:53,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T02:44:54,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552354095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:54,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552354095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:54,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T02:44:54,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:54,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:54,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:54,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/0d00a06fe6744744ac5f681b6cc3c2e6 2024-11-14T02:44:54,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/79c1527f1d9f4ce5a546911ad6cb9877 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/79c1527f1d9f4ce5a546911ad6cb9877 2024-11-14T02:44:54,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/79c1527f1d9f4ce5a546911ad6cb9877, entries=200, sequenceid=499, filesize=14.4 K 2024-11-14T02:44:54,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/c2a9497cdd1b47b59b06cc65eccabcb5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c2a9497cdd1b47b59b06cc65eccabcb5 2024-11-14T02:44:54,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c2a9497cdd1b47b59b06cc65eccabcb5, entries=150, sequenceid=499, filesize=12.0 K 2024-11-14T02:44:54,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/0d00a06fe6744744ac5f681b6cc3c2e6 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/0d00a06fe6744744ac5f681b6cc3c2e6 2024-11-14T02:44:54,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/0d00a06fe6744744ac5f681b6cc3c2e6, entries=150, sequenceid=499, filesize=12.0 K 2024-11-14T02:44:54,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T02:44:54,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:54,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:54,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:54,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 21da8e2d40bdcddc538290150d14efc8 in 850ms, sequenceid=499, compaction requested=true 2024-11-14T02:44:54,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:54,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,299 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:54,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:54,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:54,299 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:54,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:54,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:54,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:54,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:44:54,301 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:54,301 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:54,301 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/B is initiating minor compaction (all files) 2024-11-14T02:44:54,301 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/A is initiating minor compaction (all files) 2024-11-14T02:44:54,301 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/B in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:54,301 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/A in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:54,301 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/bb497c2631ee49eca465c4745b8713da, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c76b659570314f6a87d9d677ffc62272, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c2a9497cdd1b47b59b06cc65eccabcb5] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=36.8 K 2024-11-14T02:44:54,301 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e707a7eb9ed64487ae115983807db23e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/d2ce6d7adfbe40a38a16785905030ec9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/79c1527f1d9f4ce5a546911ad6cb9877] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=39.2 K 2024-11-14T02:44:54,301 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting e707a7eb9ed64487ae115983807db23e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731552291079 2024-11-14T02:44:54,301 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting bb497c2631ee49eca465c4745b8713da, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731552291079 2024-11-14T02:44:54,302 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2ce6d7adfbe40a38a16785905030ec9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731552291685 2024-11-14T02:44:54,302 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting c76b659570314f6a87d9d677ffc62272, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731552291685 2024-11-14T02:44:54,302 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79c1527f1d9f4ce5a546911ad6cb9877, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731552293320 2024-11-14T02:44:54,302 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting c2a9497cdd1b47b59b06cc65eccabcb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731552293320 2024-11-14T02:44:54,309 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#A#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:54,310 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#B#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:54,310 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/1e33fc3e2a6b4d38846da76c2aae124b is 50, key is test_row_0/A:col10/1731552293448/Put/seqid=0 2024-11-14T02:44:54,310 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/e29899f938f5401bb13a4191d56c0d67 is 50, key is test_row_0/B:col10/1731552293448/Put/seqid=0 2024-11-14T02:44:54,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742122_1298 (size=13153) 2024-11-14T02:44:54,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742121_1297 (size=13153) 2024-11-14T02:44:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T02:44:54,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T02:44:54,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:54,451 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-14T02:44:54,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:54,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:54,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:54,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:54,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:54,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:44:54,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/e61774048d0240afa47e5b998a0de832 is 50, key is test_row_1/A:col10/1731552293449/Put/seqid=0 2024-11-14T02:44:54,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742123_1299 (size=9857) 2024-11-14T02:44:54,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. as already flushing 2024-11-14T02:44:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:54,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552294635, max timestamp=1731552294646Num uniques 859; 2024-11-14T02:44:54,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552294635, max timestamp=1731552294646Num uniques 859; 2024-11-14T02:44:54,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:54,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:44:54,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731552294640, max timestamp=1731552294646Num uniques 859; 2024-11-14T02:44:54,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T02:44:54,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552354671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:54,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552354672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:54,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1731552354673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:54,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49758 deadline: 1731552354695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:54,697 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in 21da8e2d40bdcddc538290150d14efc8 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:44:54,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:54,718 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/e29899f938f5401bb13a4191d56c0d67 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/e29899f938f5401bb13a4191d56c0d67 2024-11-14T02:44:54,722 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/1e33fc3e2a6b4d38846da76c2aae124b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1e33fc3e2a6b4d38846da76c2aae124b 2024-11-14T02:44:54,724 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/B of 21da8e2d40bdcddc538290150d14efc8 into e29899f938f5401bb13a4191d56c0d67(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:54,724 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:54,724 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/B, priority=13, startTime=1731552294299; duration=0sec 2024-11-14T02:44:54,724 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:54,724 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:B 2024-11-14T02:44:54,724 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:54,725 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:54,725 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/C is initiating minor compaction (all files) 2024-11-14T02:44:54,725 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/C in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:54,725 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/d393a8dde18b4ceba995b94450179d39, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/65289f31143143959954820e7190cd81, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/0d00a06fe6744744ac5f681b6cc3c2e6] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=36.8 K 2024-11-14T02:44:54,726 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting d393a8dde18b4ceba995b94450179d39, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731552291079 2024-11-14T02:44:54,726 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 65289f31143143959954820e7190cd81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731552291685 2024-11-14T02:44:54,726 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d00a06fe6744744ac5f681b6cc3c2e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731552293320 2024-11-14T02:44:54,728 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/A of 21da8e2d40bdcddc538290150d14efc8 into 1e33fc3e2a6b4d38846da76c2aae124b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:54,728 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:54,728 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/A, priority=13, startTime=1731552294299; duration=0sec 2024-11-14T02:44:54,728 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:54,728 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:A 2024-11-14T02:44:54,733 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#C#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:54,734 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/3af07986b0384f229e714aad04ac9fdf is 50, key is test_row_0/C:col10/1731552293448/Put/seqid=0 2024-11-14T02:44:54,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742124_1300 (size=13153) 2024-11-14T02:44:54,747 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/3af07986b0384f229e714aad04ac9fdf as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3af07986b0384f229e714aad04ac9fdf 2024-11-14T02:44:54,751 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/C of 21da8e2d40bdcddc538290150d14efc8 into 3af07986b0384f229e714aad04ac9fdf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:54,751 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:54,751 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/C, priority=13, startTime=1731552294299; duration=0sec 2024-11-14T02:44:54,751 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:54,751 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:C 2024-11-14T02:44:54,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1731552354785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:54,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552354785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552354785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T02:44:54,860 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/e61774048d0240afa47e5b998a0de832 2024-11-14T02:44:54,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/33701e6b71e64d2da989f951f26512f8 is 50, key is test_row_1/B:col10/1731552293449/Put/seqid=0 2024-11-14T02:44:54,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742125_1301 (size=9857) 2024-11-14T02:44:54,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:54,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:54,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1731552354995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552354995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552354995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:55,195 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T02:44:55,269 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/33701e6b71e64d2da989f951f26512f8 2024-11-14T02:44:55,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/6f93a4857f07497b8dd146cc4e69c214 is 50, key is test_row_1/C:col10/1731552293449/Put/seqid=0 2024-11-14T02:44:55,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742126_1302 (size=9857) 2024-11-14T02:44:55,278 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/6f93a4857f07497b8dd146cc4e69c214 2024-11-14T02:44:55,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/e61774048d0240afa47e5b998a0de832 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e61774048d0240afa47e5b998a0de832 2024-11-14T02:44:55,286 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e61774048d0240afa47e5b998a0de832, entries=100, sequenceid=509, filesize=9.6 K 2024-11-14T02:44:55,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/33701e6b71e64d2da989f951f26512f8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/33701e6b71e64d2da989f951f26512f8 2024-11-14T02:44:55,290 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/33701e6b71e64d2da989f951f26512f8, entries=100, sequenceid=509, filesize=9.6 K 2024-11-14T02:44:55,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/6f93a4857f07497b8dd146cc4e69c214 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6f93a4857f07497b8dd146cc4e69c214 2024-11-14T02:44:55,295 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6f93a4857f07497b8dd146cc4e69c214, entries=100, sequenceid=509, filesize=9.6 K 2024-11-14T02:44:55,296 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=138.26 KB/141575 for 21da8e2d40bdcddc538290150d14efc8 in 845ms, sequenceid=509, compaction requested=false 2024-11-14T02:44:55,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:55,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:55,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-14T02:44:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-14T02:44:55,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-14T02:44:55,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6090 sec 2024-11-14T02:44:55,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.6140 sec 2024-11-14T02:44:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:44:55,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731552294667, max timestamp=1731552295307Num uniques 1068; 2024-11-14T02:44:55,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-14T02:44:55,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731552294667, max timestamp=1731552295307Num uniques 1068; 2024-11-14T02:44:55,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-14T02:44:55,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731552294667, max timestamp=1731552295307Num uniques 1009; 2024-11-14T02:44:55,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-14T02:44:55,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=40.25 KB heapSize=105.23 KB 2024-11-14T02:44:55,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:44:55,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:55,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:44:55,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:55,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:44:55,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:44:55,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/f7b8692524cf49da9eed5a3eb1a85374 is 50, key is test_row_0/A:col10/1731552294669/Put/seqid=0 2024-11-14T02:44:55,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742127_1303 (size=17181) 2024-11-14T02:44:55,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552295333, max timestamp=1731552295340Num uniques 859; 2024-11-14T02:44:55,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:44:55,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552295333, max timestamp=1731552295340Num uniques 859; 2024-11-14T02:44:55,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:44:55,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552295333, max timestamp=1731552295340Num uniques 859; 2024-11-14T02:44:55,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:44:55,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731552295358, max timestamp=1731552295373Num uniques 1018; 2024-11-14T02:44:55,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T02:44:55,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731552295358, max timestamp=1731552295373Num uniques 1018; 2024-11-14T02:44:55,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T02:44:55,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731552295358, max timestamp=1731552295373Num uniques 1018; 2024-11-14T02:44:55,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T02:44:55,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1731552355404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:55,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731552295402, max timestamp=1731552295409Num uniques 1077; 2024-11-14T02:44:55,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-14T02:44:55,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731552295402, max timestamp=1731552295409Num uniques 1077; 2024-11-14T02:44:55,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-14T02:44:55,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731552295402, max timestamp=1731552295409Num uniques 1077; 2024-11-14T02:44:55,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-14T02:44:55,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731552295449, max timestamp=1731552295458Num uniques 1036; 2024-11-14T02:44:55,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-14T02:44:55,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731552295449, max timestamp=1731552295458Num uniques 1036; 2024-11-14T02:44:55,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-14T02:44:55,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731552295449, max timestamp=1731552295458Num uniques 1036; 2024-11-14T02:44:55,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-14T02:44:55,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731552295494, max timestamp=1731552295505Num uniques 1045; 2024-11-14T02:44:55,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731552295494, max timestamp=1731552295505Num uniques 1045; 2024-11-14T02:44:55,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731552295494, max timestamp=1731552295505Num uniques 1045; 2024-11-14T02:44:55,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-14T02:44:55,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-14T02:44:55,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-14T02:44:55,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 351 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49724 deadline: 1731552355542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:55,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731552295524, max timestamp=1731552295546Num uniques 1104; 2024-11-14T02:44:55,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-14T02:44:55,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731552295524, max timestamp=1731552295546Num uniques 1104; 2024-11-14T02:44:55,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-14T02:44:55,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731552295524, max timestamp=1731552295546Num uniques 1104; 2024-11-14T02:44:55,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-14T02:44:55,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:44:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49700 deadline: 1731552355612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:44:55,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:44:55,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:44:55,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731552295576, max timestamp=1731552295615Num uniques 1063; 2024-11-14T02:44:55,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-14T02:44:55,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731552295576, max timestamp=1731552295615Num uniques 1063; 2024-11-14T02:44:55,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-14T02:44:55,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731552295576, max timestamp=1731552295615Num uniques 1063; 2024-11-14T02:44:55,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-14T02:44:55,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731552295670, max timestamp=1731552295685Num uniques 1072; 2024-11-14T02:44:55,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-14T02:44:55,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731552295670, max timestamp=1731552295685Num uniques 1072; 2024-11-14T02:44:55,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-14T02:44:55,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731552295670, max timestamp=1731552295685Num uniques 1072; 2024-11-14T02:44:55,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-14T02:44:55,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/f7b8692524cf49da9eed5a3eb1a85374 2024-11-14T02:44:55,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/f480354f78ec4146bbcb42a9e66f2d06 is 50, key is test_row_0/B:col10/1731552294669/Put/seqid=0 2024-11-14T02:44:55,726 INFO [Thread-1062 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1062. 2024-11-14T02:44:55,726 DEBUG [Thread-1062 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:55,726 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,726 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:55,726 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-11-14T02:44:55,726 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:55,726 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,726 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:55,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742128_1304 (size=12301) 2024-11-14T02:44:55,728 INFO [Thread-1049 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1049. 2024-11-14T02:44:55,728 DEBUG [Thread-1049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:55,728 DEBUG [Thread-1049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,729 DEBUG [Thread-1049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,729 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:55,731 INFO [Thread-1060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1060. 2024-11-14T02:44:55,731 DEBUG [Thread-1060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:55,731 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,731 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:55,731 INFO [Thread-1051 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1051. 2024-11-14T02:44:55,731 DEBUG [Thread-1051 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:55,731 DEBUG [Thread-1051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,731 DEBUG [Thread-1051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:55,732 INFO [Thread-1058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1058. 2024-11-14T02:44:55,732 DEBUG [Thread-1058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:55,732 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,732 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:55,732 INFO [Thread-1045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1045. 2024-11-14T02:44:55,732 DEBUG [Thread-1045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:55,732 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,732 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:55,734 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-11-14T02:44:55,735 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:55,735 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,735 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,735 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T02:44:55,836 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:44:55,887 INFO [Thread-1047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1047. 2024-11-14T02:44:55,887 DEBUG [Thread-1047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:44:55,887 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,887 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:44:55,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:44:56,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/f480354f78ec4146bbcb42a9e66f2d06 2024-11-14T02:44:56,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/f9a521b3ff2d48e2814332de86c00fd7 is 50, key is test_row_0/C:col10/1731552294669/Put/seqid=0 2024-11-14T02:44:56,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742129_1305 (size=12301) 2024-11-14T02:44:56,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/f9a521b3ff2d48e2814332de86c00fd7 2024-11-14T02:44:56,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/f7b8692524cf49da9eed5a3eb1a85374 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f7b8692524cf49da9eed5a3eb1a85374 2024-11-14T02:44:56,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f7b8692524cf49da9eed5a3eb1a85374, entries=250, sequenceid=552, filesize=16.8 K 2024-11-14T02:44:56,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/f480354f78ec4146bbcb42a9e66f2d06 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/f480354f78ec4146bbcb42a9e66f2d06 2024-11-14T02:44:56,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/f480354f78ec4146bbcb42a9e66f2d06, entries=150, sequenceid=552, filesize=12.0 K 2024-11-14T02:44:56,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/f9a521b3ff2d48e2814332de86c00fd7 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/f9a521b3ff2d48e2814332de86c00fd7 2024-11-14T02:44:56,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/f9a521b3ff2d48e2814332de86c00fd7, entries=150, sequenceid=552, filesize=12.0 K 2024-11-14T02:44:56,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~105.19 KB/107712, currentSize=114.05 KB/116790 for 21da8e2d40bdcddc538290150d14efc8 in 1301ms, sequenceid=552, compaction requested=true 2024-11-14T02:44:56,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:56,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:44:56,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:56,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:44:56,613 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:56,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:56,613 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:56,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21da8e2d40bdcddc538290150d14efc8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:44:56,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:56,614 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:56,614 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40191 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:56,614 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/A is initiating minor compaction (all files) 2024-11-14T02:44:56,614 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/B is initiating minor compaction (all files) 2024-11-14T02:44:56,614 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/B in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:56,614 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/A in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:56,614 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1e33fc3e2a6b4d38846da76c2aae124b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e61774048d0240afa47e5b998a0de832, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f7b8692524cf49da9eed5a3eb1a85374] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=39.2 K 2024-11-14T02:44:56,614 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/e29899f938f5401bb13a4191d56c0d67, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/33701e6b71e64d2da989f951f26512f8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/f480354f78ec4146bbcb42a9e66f2d06] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=34.5 K 2024-11-14T02:44:56,614 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e33fc3e2a6b4d38846da76c2aae124b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731552293320 2024-11-14T02:44:56,614 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting e29899f938f5401bb13a4191d56c0d67, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731552293320 2024-11-14T02:44:56,614 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting e61774048d0240afa47e5b998a0de832, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1731552293449 2024-11-14T02:44:56,614 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 33701e6b71e64d2da989f951f26512f8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1731552293449 2024-11-14T02:44:56,615 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting f480354f78ec4146bbcb42a9e66f2d06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1731552294669 2024-11-14T02:44:56,615 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7b8692524cf49da9eed5a3eb1a85374, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1731552294669 2024-11-14T02:44:56,620 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#B#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:56,621 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/46d7b839ef7849b089280a97d7097865 is 50, key is test_row_0/B:col10/1731552294669/Put/seqid=0 2024-11-14T02:44:56,622 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#A#compaction#264 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:56,623 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/703459f610b940778580de8469091a61 is 50, key is test_row_0/A:col10/1731552294669/Put/seqid=0 2024-11-14T02:44:56,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742130_1306 (size=13255) 2024-11-14T02:44:56,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742131_1307 (size=13255) 2024-11-14T02:44:57,033 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/46d7b839ef7849b089280a97d7097865 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/46d7b839ef7849b089280a97d7097865 2024-11-14T02:44:57,034 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/703459f610b940778580de8469091a61 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/703459f610b940778580de8469091a61 2024-11-14T02:44:57,040 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/B of 21da8e2d40bdcddc538290150d14efc8 into 46d7b839ef7849b089280a97d7097865(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:57,040 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/A of 21da8e2d40bdcddc538290150d14efc8 into 703459f610b940778580de8469091a61(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:57,040 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:57,040 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:57,040 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/B, priority=13, startTime=1731552296613; duration=0sec 2024-11-14T02:44:57,040 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/A, priority=13, startTime=1731552296612; duration=0sec 2024-11-14T02:44:57,040 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:57,040 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:A 2024-11-14T02:44:57,040 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:44:57,040 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:B 2024-11-14T02:44:57,041 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:44:57,042 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:44:57,042 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 21da8e2d40bdcddc538290150d14efc8/C is initiating minor compaction (all files) 2024-11-14T02:44:57,042 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21da8e2d40bdcddc538290150d14efc8/C in TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:44:57,042 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3af07986b0384f229e714aad04ac9fdf, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6f93a4857f07497b8dd146cc4e69c214, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/f9a521b3ff2d48e2814332de86c00fd7] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp, totalSize=34.5 K 2024-11-14T02:44:57,042 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3af07986b0384f229e714aad04ac9fdf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731552293320 2024-11-14T02:44:57,042 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f93a4857f07497b8dd146cc4e69c214, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1731552293449 2024-11-14T02:44:57,043 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9a521b3ff2d48e2814332de86c00fd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1731552294669 2024-11-14T02:44:57,049 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21da8e2d40bdcddc538290150d14efc8#C#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:44:57,049 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/cfa505493b1d4e0abc00b6ea868f9923 is 50, key is test_row_0/C:col10/1731552294669/Put/seqid=0 2024-11-14T02:44:57,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742132_1308 (size=13255) 2024-11-14T02:44:57,463 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/cfa505493b1d4e0abc00b6ea868f9923 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/cfa505493b1d4e0abc00b6ea868f9923 2024-11-14T02:44:57,472 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21da8e2d40bdcddc538290150d14efc8/C of 21da8e2d40bdcddc538290150d14efc8 into cfa505493b1d4e0abc00b6ea868f9923(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:44:57,472 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:44:57,472 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8., storeName=21da8e2d40bdcddc538290150d14efc8/C, priority=13, startTime=1731552296613; duration=0sec 2024-11-14T02:44:57,472 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:44:57,472 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21da8e2d40bdcddc538290150d14efc8:C 2024-11-14T02:45:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:45:04,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21da8e2d40bdcddc538290150d14efc8 3/3 column families, dataSize=120.76 KB heapSize=343.92 KB 2024-11-14T02:45:04,720 INFO [Thread-1043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1043. 2024-11-14T02:45:04,721 DEBUG [Thread-1043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:04,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=A 2024-11-14T02:45:04,721 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:04,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:04,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=B 2024-11-14T02:45:04,721 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:04,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:04,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:04,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21da8e2d40bdcddc538290150d14efc8, store=C 2024-11-14T02:45:04,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:04,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T02:45:04,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-14T02:45:04,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 163 2024-11-14T02:45:04,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-14T02:45:04,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 305 2024-11-14T02:45:04,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-14T02:45:04,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T02:45:04,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3266 2024-11-14T02:45:04,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3041 2024-11-14T02:45:04,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3043 2024-11-14T02:45:04,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3247 2024-11-14T02:45:04,722 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3029 2024-11-14T02:45:04,723 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T02:45:04,723 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:45:04,723 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:45:04,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:04,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:04,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:04,725 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T02:45:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T02:45:04,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/68771a3e1a644469a6041bc4df327758 is 50, key is test_row_0/A:col10/1731552295723/Put/seqid=0 2024-11-14T02:45:04,729 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552304729"}]},"ts":"1731552304729"} 2024-11-14T02:45:04,730 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T02:45:04,730 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T02:45:04,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:45:04,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742133_1309 (size=12301) 2024-11-14T02:45:04,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21da8e2d40bdcddc538290150d14efc8, UNASSIGN}] 2024-11-14T02:45:04,734 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21da8e2d40bdcddc538290150d14efc8, UNASSIGN 2024-11-14T02:45:04,735 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=21da8e2d40bdcddc538290150d14efc8, regionState=CLOSING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:04,735 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=e5dc8b5cf5e9,46387,1731552206828, table=TestAcidGuarantees, region=21da8e2d40bdcddc538290150d14efc8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-14T02:45:04,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21da8e2d40bdcddc538290150d14efc8, UNASSIGN because future has completed 2024-11-14T02:45:04,736 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:45:04,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:45:04,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T02:45:04,888 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(122): Close 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:45:04,888 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:45:04,888 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1722): Closing 21da8e2d40bdcddc538290150d14efc8, disabling compactions & flushes 2024-11-14T02:45:04,888 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:45:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T02:45:05,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=710 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/68771a3e1a644469a6041bc4df327758 2024-11-14T02:45:05,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/5c4cbbaee3594de89018835755c07053 is 50, key is test_row_0/B:col10/1731552295723/Put/seqid=0 2024-11-14T02:45:05,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742134_1310 (size=12301) 2024-11-14T02:45:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T02:45:05,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=710 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/5c4cbbaee3594de89018835755c07053 2024-11-14T02:45:05,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/7120a5e73dd4465eafedf3e36f76e6a4 is 50, key is test_row_0/C:col10/1731552295723/Put/seqid=0 2024-11-14T02:45:05,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742135_1311 (size=12301) 2024-11-14T02:45:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T02:45:05,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=710 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/7120a5e73dd4465eafedf3e36f76e6a4 2024-11-14T02:45:05,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/A/68771a3e1a644469a6041bc4df327758 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/68771a3e1a644469a6041bc4df327758 2024-11-14T02:45:05,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/68771a3e1a644469a6041bc4df327758, entries=150, sequenceid=710, filesize=12.0 K 2024-11-14T02:45:05,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/B/5c4cbbaee3594de89018835755c07053 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/5c4cbbaee3594de89018835755c07053 2024-11-14T02:45:05,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/5c4cbbaee3594de89018835755c07053, entries=150, sequenceid=710, filesize=12.0 K 2024-11-14T02:45:05,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/.tmp/C/7120a5e73dd4465eafedf3e36f76e6a4 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/7120a5e73dd4465eafedf3e36f76e6a4 2024-11-14T02:45:05,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/7120a5e73dd4465eafedf3e36f76e6a4, entries=150, sequenceid=710, filesize=12.0 K 2024-11-14T02:45:05,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~343.88 KB/352128, currentSize=0 B/0 for 21da8e2d40bdcddc538290150d14efc8 in 1276ms, sequenceid=710, compaction requested=false 2024-11-14T02:45:05,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21da8e2d40bdcddc538290150d14efc8: 2024-11-14T02:45:05,997 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:45:05,997 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:45:05,997 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. after waiting 0 ms 2024-11-14T02:45:05,997 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:45:05,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/7585ab9dc25e4e8ab304b145211db174, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/5548fe3cfcb74869bf6cd4ab53d0157c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f7342d0b76ac4cd68e439aa5efe8c25b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/abb4ca26593d4910a51a556129029ef4, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f018de12f9a54f539b4c6148703fb578, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/4972db8175844514bd6c11dc3064522e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e170a5eb5fc44198864f52402ddd2bde, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/644651748b6a403bb049eaf289f5f29f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/72106d00f331486181e526e80d03d9bd, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/26dcc0358ec540d38ce61fd3093d6fd3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f1d8b35202d849d8bc721f2386c756c3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/cf7692a3c0f24e6b8758f6aaaff6a267, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1e08da3d12ec4bca904c41ed5b3285bd, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1be5b490573b4caf8cb46a28fc7c55d0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/63f265c5cf86436a8d2bdcf1f811f99c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/042a39d8c5a747a4bbb005daed7489b0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/73ef4b9b60cc43d1900cf77baece76d2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/5c2f0ad78c6d47ecb31d2163dbdf76e5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/4ad8aacd03e34a638b64c181b2e29e6c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/6bdc30d9ed244d3aa07ba8098b3e8b19, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f3ab64084d914f79ae3139ccf3b3bad3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1ef290d2c00d4b58889b6c754163b50c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e707a7eb9ed64487ae115983807db23e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/d2ce6d7adfbe40a38a16785905030ec9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/79c1527f1d9f4ce5a546911ad6cb9877, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1e33fc3e2a6b4d38846da76c2aae124b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e61774048d0240afa47e5b998a0de832, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f7b8692524cf49da9eed5a3eb1a85374] to archive 2024-11-14T02:45:05,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:45:06,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/7585ab9dc25e4e8ab304b145211db174 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/7585ab9dc25e4e8ab304b145211db174 2024-11-14T02:45:06,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/5548fe3cfcb74869bf6cd4ab53d0157c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/5548fe3cfcb74869bf6cd4ab53d0157c 2024-11-14T02:45:06,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f7342d0b76ac4cd68e439aa5efe8c25b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f7342d0b76ac4cd68e439aa5efe8c25b 2024-11-14T02:45:06,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/abb4ca26593d4910a51a556129029ef4 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/abb4ca26593d4910a51a556129029ef4 2024-11-14T02:45:06,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f018de12f9a54f539b4c6148703fb578 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f018de12f9a54f539b4c6148703fb578 2024-11-14T02:45:06,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/4972db8175844514bd6c11dc3064522e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/4972db8175844514bd6c11dc3064522e 2024-11-14T02:45:06,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e170a5eb5fc44198864f52402ddd2bde to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e170a5eb5fc44198864f52402ddd2bde 2024-11-14T02:45:06,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/644651748b6a403bb049eaf289f5f29f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/644651748b6a403bb049eaf289f5f29f 2024-11-14T02:45:06,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/72106d00f331486181e526e80d03d9bd to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/72106d00f331486181e526e80d03d9bd 2024-11-14T02:45:06,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/26dcc0358ec540d38ce61fd3093d6fd3 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/26dcc0358ec540d38ce61fd3093d6fd3 2024-11-14T02:45:06,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f1d8b35202d849d8bc721f2386c756c3 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f1d8b35202d849d8bc721f2386c756c3 2024-11-14T02:45:06,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/cf7692a3c0f24e6b8758f6aaaff6a267 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/cf7692a3c0f24e6b8758f6aaaff6a267 2024-11-14T02:45:06,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1e08da3d12ec4bca904c41ed5b3285bd to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1e08da3d12ec4bca904c41ed5b3285bd 2024-11-14T02:45:06,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1be5b490573b4caf8cb46a28fc7c55d0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1be5b490573b4caf8cb46a28fc7c55d0 2024-11-14T02:45:06,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/63f265c5cf86436a8d2bdcf1f811f99c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/63f265c5cf86436a8d2bdcf1f811f99c 2024-11-14T02:45:06,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/042a39d8c5a747a4bbb005daed7489b0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/042a39d8c5a747a4bbb005daed7489b0 2024-11-14T02:45:06,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/73ef4b9b60cc43d1900cf77baece76d2 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/73ef4b9b60cc43d1900cf77baece76d2 2024-11-14T02:45:06,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/5c2f0ad78c6d47ecb31d2163dbdf76e5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/5c2f0ad78c6d47ecb31d2163dbdf76e5 2024-11-14T02:45:06,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/4ad8aacd03e34a638b64c181b2e29e6c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/4ad8aacd03e34a638b64c181b2e29e6c 2024-11-14T02:45:06,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/6bdc30d9ed244d3aa07ba8098b3e8b19 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/6bdc30d9ed244d3aa07ba8098b3e8b19 2024-11-14T02:45:06,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f3ab64084d914f79ae3139ccf3b3bad3 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f3ab64084d914f79ae3139ccf3b3bad3 2024-11-14T02:45:06,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1ef290d2c00d4b58889b6c754163b50c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1ef290d2c00d4b58889b6c754163b50c 2024-11-14T02:45:06,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e707a7eb9ed64487ae115983807db23e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e707a7eb9ed64487ae115983807db23e 2024-11-14T02:45:06,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/d2ce6d7adfbe40a38a16785905030ec9 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/d2ce6d7adfbe40a38a16785905030ec9 2024-11-14T02:45:06,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/79c1527f1d9f4ce5a546911ad6cb9877 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/79c1527f1d9f4ce5a546911ad6cb9877 2024-11-14T02:45:06,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1e33fc3e2a6b4d38846da76c2aae124b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/1e33fc3e2a6b4d38846da76c2aae124b 2024-11-14T02:45:06,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e61774048d0240afa47e5b998a0de832 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/e61774048d0240afa47e5b998a0de832 2024-11-14T02:45:06,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f7b8692524cf49da9eed5a3eb1a85374 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/f7b8692524cf49da9eed5a3eb1a85374 2024-11-14T02:45:06,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/355ec4b861bc4fbe8c5e9e257843cb1e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/007e9cdcf3d24adf8271a5db9693c1cb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/8f3d88c51d054fbfb2352c11aed28494, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/2294e862b6ab48a6aba3857006f76f43, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/5726f86a6fe446d78069d8131f1e4379, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/05a246e0cc1b45399b276550d0ae9286, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/27c3491380814481bd4c44945488a59a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/4892a92fcd7f4020b7353cba0c2a0e55, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/12e0bf8716bd452b8ce8d9a4e2f87c96, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/588763612e1c478581b10ca4376d8c62, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ade6b1f42f574eb1aa454bbc1cd9faa2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/f2248093a18d4aa6a1fd7ff4789aa25e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/68ee76ed7576492e8915af48041b10a8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/dcbf32c85d934f22ba7fe41cd97cf1b1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ab18cae7292f470d911721ca1888500b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/7d71a58e013f454896f3c49cb110f658, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c5c66e6c879f46d6aa58dd6b472a1faa, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/be19b12086784830916be20a7eb82caf, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/b3e856066f95497ebf1300dc975969d7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/13820fb862ae458287a490bba814cd5c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/2c686021e5c24d8499dae37efa18efea, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/bb497c2631ee49eca465c4745b8713da, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ca3695839d8440f898d37cd679cdc253, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c76b659570314f6a87d9d677ffc62272, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/e29899f938f5401bb13a4191d56c0d67, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c2a9497cdd1b47b59b06cc65eccabcb5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/33701e6b71e64d2da989f951f26512f8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/f480354f78ec4146bbcb42a9e66f2d06] to archive 2024-11-14T02:45:06,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:45:06,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/355ec4b861bc4fbe8c5e9e257843cb1e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/355ec4b861bc4fbe8c5e9e257843cb1e 2024-11-14T02:45:06,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/007e9cdcf3d24adf8271a5db9693c1cb to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/007e9cdcf3d24adf8271a5db9693c1cb 2024-11-14T02:45:06,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/8f3d88c51d054fbfb2352c11aed28494 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/8f3d88c51d054fbfb2352c11aed28494 2024-11-14T02:45:06,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/2294e862b6ab48a6aba3857006f76f43 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/2294e862b6ab48a6aba3857006f76f43 2024-11-14T02:45:06,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/5726f86a6fe446d78069d8131f1e4379 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/5726f86a6fe446d78069d8131f1e4379 2024-11-14T02:45:06,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/05a246e0cc1b45399b276550d0ae9286 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/05a246e0cc1b45399b276550d0ae9286 2024-11-14T02:45:06,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/27c3491380814481bd4c44945488a59a to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/27c3491380814481bd4c44945488a59a 2024-11-14T02:45:06,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/4892a92fcd7f4020b7353cba0c2a0e55 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/4892a92fcd7f4020b7353cba0c2a0e55 2024-11-14T02:45:06,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/12e0bf8716bd452b8ce8d9a4e2f87c96 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/12e0bf8716bd452b8ce8d9a4e2f87c96 2024-11-14T02:45:06,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/588763612e1c478581b10ca4376d8c62 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/588763612e1c478581b10ca4376d8c62 2024-11-14T02:45:06,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ade6b1f42f574eb1aa454bbc1cd9faa2 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ade6b1f42f574eb1aa454bbc1cd9faa2 2024-11-14T02:45:06,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/f2248093a18d4aa6a1fd7ff4789aa25e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/f2248093a18d4aa6a1fd7ff4789aa25e 2024-11-14T02:45:06,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/68ee76ed7576492e8915af48041b10a8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/68ee76ed7576492e8915af48041b10a8 2024-11-14T02:45:06,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/dcbf32c85d934f22ba7fe41cd97cf1b1 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/dcbf32c85d934f22ba7fe41cd97cf1b1 2024-11-14T02:45:06,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ab18cae7292f470d911721ca1888500b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ab18cae7292f470d911721ca1888500b 2024-11-14T02:45:06,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/7d71a58e013f454896f3c49cb110f658 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/7d71a58e013f454896f3c49cb110f658 2024-11-14T02:45:06,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c5c66e6c879f46d6aa58dd6b472a1faa to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c5c66e6c879f46d6aa58dd6b472a1faa 2024-11-14T02:45:06,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/be19b12086784830916be20a7eb82caf to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/be19b12086784830916be20a7eb82caf 2024-11-14T02:45:06,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/b3e856066f95497ebf1300dc975969d7 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/b3e856066f95497ebf1300dc975969d7 2024-11-14T02:45:06,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/13820fb862ae458287a490bba814cd5c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/13820fb862ae458287a490bba814cd5c 2024-11-14T02:45:06,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/2c686021e5c24d8499dae37efa18efea to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/2c686021e5c24d8499dae37efa18efea 2024-11-14T02:45:06,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/bb497c2631ee49eca465c4745b8713da to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/bb497c2631ee49eca465c4745b8713da 2024-11-14T02:45:06,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ca3695839d8440f898d37cd679cdc253 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/ca3695839d8440f898d37cd679cdc253 2024-11-14T02:45:06,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c76b659570314f6a87d9d677ffc62272 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c76b659570314f6a87d9d677ffc62272 2024-11-14T02:45:06,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/e29899f938f5401bb13a4191d56c0d67 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/e29899f938f5401bb13a4191d56c0d67 2024-11-14T02:45:06,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c2a9497cdd1b47b59b06cc65eccabcb5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/c2a9497cdd1b47b59b06cc65eccabcb5 2024-11-14T02:45:06,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/33701e6b71e64d2da989f951f26512f8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/33701e6b71e64d2da989f951f26512f8 2024-11-14T02:45:06,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/f480354f78ec4146bbcb42a9e66f2d06 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/f480354f78ec4146bbcb42a9e66f2d06 2024-11-14T02:45:06,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/ebf9cfc3fb454581bacef027ead29d73, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/62e35067959d4195b8e9fe91627c4f00, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3c67a92505894acbb5ce20f1e5acd42e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/5e9f065581cd43dfae7b38d9719d647c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/815ddd9208e84d379ce18de49ae85d70, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/5ee4b821bd9b4c8fa3d5d2d226482ce3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/0bfaa9bddfa34725a09bdc2e0634609d, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/02db01f26d6649499d3e9c7f4053fe3f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/77c310698ed14ef4b08d6327d9059509, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/759e3c0aeff34939a8dcc84669f8bfe5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/68a84d2bd329429f9251394c9c7d57a0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/da3ad4e1ac3f41ecb1c249d7f7f38c44, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/b9b0f3a9165046a696a0f9525e07f4a2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/59aa934841a14cb6a1e7b1a6058073bf, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3315783654904ed284870f15573491dc, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6655f3f164f64960be22a406a1a3d1e7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/b56c609b68ba44b482bfe882067d10fc, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/85f2a88fbb2042818438d8fa4224f58a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/eb3dcba7e91d40fe8880bdb1c2703ed4, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6ccac307e997448e90f13787b83f8348, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/74260c3f6c5a4ccbb6206cd0ad97e956, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/d393a8dde18b4ceba995b94450179d39, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/c36f2c9b101f45178b9d2656e219c826, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/65289f31143143959954820e7190cd81, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3af07986b0384f229e714aad04ac9fdf, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/0d00a06fe6744744ac5f681b6cc3c2e6, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6f93a4857f07497b8dd146cc4e69c214, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/f9a521b3ff2d48e2814332de86c00fd7] to archive 2024-11-14T02:45:06,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:45:06,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/ebf9cfc3fb454581bacef027ead29d73 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/ebf9cfc3fb454581bacef027ead29d73 2024-11-14T02:45:06,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/62e35067959d4195b8e9fe91627c4f00 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/62e35067959d4195b8e9fe91627c4f00 2024-11-14T02:45:06,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3c67a92505894acbb5ce20f1e5acd42e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3c67a92505894acbb5ce20f1e5acd42e 2024-11-14T02:45:06,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/5e9f065581cd43dfae7b38d9719d647c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/5e9f065581cd43dfae7b38d9719d647c 2024-11-14T02:45:06,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/815ddd9208e84d379ce18de49ae85d70 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/815ddd9208e84d379ce18de49ae85d70 2024-11-14T02:45:06,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/5ee4b821bd9b4c8fa3d5d2d226482ce3 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/5ee4b821bd9b4c8fa3d5d2d226482ce3 2024-11-14T02:45:06,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/0bfaa9bddfa34725a09bdc2e0634609d to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/0bfaa9bddfa34725a09bdc2e0634609d 2024-11-14T02:45:06,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/02db01f26d6649499d3e9c7f4053fe3f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/02db01f26d6649499d3e9c7f4053fe3f 2024-11-14T02:45:06,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/77c310698ed14ef4b08d6327d9059509 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/77c310698ed14ef4b08d6327d9059509 2024-11-14T02:45:06,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/759e3c0aeff34939a8dcc84669f8bfe5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/759e3c0aeff34939a8dcc84669f8bfe5 2024-11-14T02:45:06,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/68a84d2bd329429f9251394c9c7d57a0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/68a84d2bd329429f9251394c9c7d57a0 2024-11-14T02:45:06,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/da3ad4e1ac3f41ecb1c249d7f7f38c44 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/da3ad4e1ac3f41ecb1c249d7f7f38c44 2024-11-14T02:45:06,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/b9b0f3a9165046a696a0f9525e07f4a2 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/b9b0f3a9165046a696a0f9525e07f4a2 2024-11-14T02:45:06,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/59aa934841a14cb6a1e7b1a6058073bf to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/59aa934841a14cb6a1e7b1a6058073bf 2024-11-14T02:45:06,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3315783654904ed284870f15573491dc to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3315783654904ed284870f15573491dc 2024-11-14T02:45:06,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6655f3f164f64960be22a406a1a3d1e7 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6655f3f164f64960be22a406a1a3d1e7 2024-11-14T02:45:06,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/b56c609b68ba44b482bfe882067d10fc to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/b56c609b68ba44b482bfe882067d10fc 2024-11-14T02:45:06,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/85f2a88fbb2042818438d8fa4224f58a to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/85f2a88fbb2042818438d8fa4224f58a 2024-11-14T02:45:06,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/eb3dcba7e91d40fe8880bdb1c2703ed4 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/eb3dcba7e91d40fe8880bdb1c2703ed4 2024-11-14T02:45:06,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6ccac307e997448e90f13787b83f8348 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6ccac307e997448e90f13787b83f8348 2024-11-14T02:45:06,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/74260c3f6c5a4ccbb6206cd0ad97e956 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/74260c3f6c5a4ccbb6206cd0ad97e956 2024-11-14T02:45:06,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/d393a8dde18b4ceba995b94450179d39 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/d393a8dde18b4ceba995b94450179d39 2024-11-14T02:45:06,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/c36f2c9b101f45178b9d2656e219c826 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/c36f2c9b101f45178b9d2656e219c826 2024-11-14T02:45:06,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/65289f31143143959954820e7190cd81 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/65289f31143143959954820e7190cd81 2024-11-14T02:45:06,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3af07986b0384f229e714aad04ac9fdf to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/3af07986b0384f229e714aad04ac9fdf 2024-11-14T02:45:06,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/0d00a06fe6744744ac5f681b6cc3c2e6 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/0d00a06fe6744744ac5f681b6cc3c2e6 2024-11-14T02:45:06,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6f93a4857f07497b8dd146cc4e69c214 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/6f93a4857f07497b8dd146cc4e69c214 2024-11-14T02:45:06,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/f9a521b3ff2d48e2814332de86c00fd7 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/f9a521b3ff2d48e2814332de86c00fd7 2024-11-14T02:45:06,079 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/recovered.edits/713.seqid, newMaxSeqId=713, maxSeqId=1 2024-11-14T02:45:06,079 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8. 2024-11-14T02:45:06,079 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1676): Region close journal for 21da8e2d40bdcddc538290150d14efc8: Waiting for close lock at 1731552304888Running coprocessor pre-close hooks at 1731552304888Disabling compacts and flushes for region at 1731552304888Disabling writes for close at 1731552305997 (+1109 ms)Writing region close event to WAL at 1731552306075 (+78 ms)Running coprocessor post-close hooks at 1731552306079 (+4 ms)Closed at 1731552306079 2024-11-14T02:45:06,082 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(157): Closed 21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:45:06,082 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=21da8e2d40bdcddc538290150d14efc8, regionState=CLOSED 2024-11-14T02:45:06,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:45:06,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-14T02:45:06,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseRegionProcedure 21da8e2d40bdcddc538290150d14efc8, server=e5dc8b5cf5e9,46387,1731552206828 in 1.3480 sec 2024-11-14T02:45:06,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-11-14T02:45:06,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21da8e2d40bdcddc538290150d14efc8, UNASSIGN in 1.3530 sec 2024-11-14T02:45:06,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-14T02:45:06,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3570 sec 2024-11-14T02:45:06,090 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552306090"}]},"ts":"1731552306090"} 2024-11-14T02:45:06,092 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T02:45:06,092 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T02:45:06,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3670 sec 2024-11-14T02:45:06,490 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T02:45:06,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T02:45:06,887 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:06,888 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T02:45:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:06,893 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:06,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T02:45:06,895 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:06,900 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:45:06,902 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/recovered.edits] 2024-11-14T02:45:06,905 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/68771a3e1a644469a6041bc4df327758 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/68771a3e1a644469a6041bc4df327758 2024-11-14T02:45:06,907 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/703459f610b940778580de8469091a61 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/A/703459f610b940778580de8469091a61 2024-11-14T02:45:06,909 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/46d7b839ef7849b089280a97d7097865 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/46d7b839ef7849b089280a97d7097865 2024-11-14T02:45:06,910 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/5c4cbbaee3594de89018835755c07053 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/B/5c4cbbaee3594de89018835755c07053 2024-11-14T02:45:06,914 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/7120a5e73dd4465eafedf3e36f76e6a4 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/7120a5e73dd4465eafedf3e36f76e6a4 2024-11-14T02:45:06,915 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/cfa505493b1d4e0abc00b6ea868f9923 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/C/cfa505493b1d4e0abc00b6ea868f9923 2024-11-14T02:45:06,919 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/recovered.edits/713.seqid to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8/recovered.edits/713.seqid 2024-11-14T02:45:06,920 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/21da8e2d40bdcddc538290150d14efc8 2024-11-14T02:45:06,920 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T02:45:06,923 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:06,927 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T02:45:06,929 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T02:45:06,931 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:06,931 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T02:45:06,931 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731552306931"}]},"ts":"9223372036854775807"} 2024-11-14T02:45:06,933 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T02:45:06,933 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 21da8e2d40bdcddc538290150d14efc8, NAME => 'TestAcidGuarantees,,1731552273515.21da8e2d40bdcddc538290150d14efc8.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T02:45:06,933 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T02:45:06,934 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731552306933"}]},"ts":"9223372036854775807"} 2024-11-14T02:45:06,936 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T02:45:06,937 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:06,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-14T02:45:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T02:45:07,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T02:45:07,005 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:07,015 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 252), OpenFileDescriptor=447 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 403), ProcessCount=11 (was 11), AvailableMemoryMB=9828 (was 9977) 2024-11-14T02:45:07,023 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=9827 2024-11-14T02:45:07,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:45:07,025 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:45:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:07,026 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T02:45:07,026 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 79 2024-11-14T02:45:07,026 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-14T02:45:07,027 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T02:45:07,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742136_1312 (size=1061) 2024-11-14T02:45:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-14T02:45:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-14T02:45:07,436 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1 2024-11-14T02:45:07,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742137_1313 (size=53) 2024-11-14T02:45:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-14T02:45:07,848 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:45:07,848 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ae5c683ab910cad12ebb3cb2593f891b, disabling compactions & flushes 2024-11-14T02:45:07,848 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:07,848 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:07,848 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. after waiting 0 ms 2024-11-14T02:45:07,848 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:07,848 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:07,848 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ae5c683ab910cad12ebb3cb2593f891b: Waiting for close lock at 1731552307848Disabling compacts and flushes for region at 1731552307848Disabling writes for close at 1731552307848Writing region close event to WAL at 1731552307848Closed at 1731552307848 2024-11-14T02:45:07,851 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T02:45:07,852 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731552307851"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731552307851"}]},"ts":"1731552307851"} 2024-11-14T02:45:07,855 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T02:45:07,857 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T02:45:07,857 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552307857"}]},"ts":"1731552307857"} 2024-11-14T02:45:07,860 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T02:45:07,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b, ASSIGN}] 2024-11-14T02:45:07,861 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b, ASSIGN 2024-11-14T02:45:07,862 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b, ASSIGN; state=OFFLINE, location=e5dc8b5cf5e9,46387,1731552206828; forceNewPlan=false, retain=false 2024-11-14T02:45:08,013 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=ae5c683ab910cad12ebb3cb2593f891b, regionState=OPENING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:08,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b, ASSIGN because future has completed 2024-11-14T02:45:08,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:45:08,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-14T02:45:08,177 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:08,177 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:45:08,178 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:08,178 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:45:08,178 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:08,178 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:08,180 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:08,181 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:45:08,182 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae5c683ab910cad12ebb3cb2593f891b columnFamilyName A 2024-11-14T02:45:08,182 DEBUG [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:08,182 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.HStore(327): Store=ae5c683ab910cad12ebb3cb2593f891b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:08,182 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:08,183 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:45:08,183 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae5c683ab910cad12ebb3cb2593f891b columnFamilyName B 2024-11-14T02:45:08,183 DEBUG [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:08,183 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.HStore(327): Store=ae5c683ab910cad12ebb3cb2593f891b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:08,183 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:08,184 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:45:08,184 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae5c683ab910cad12ebb3cb2593f891b columnFamilyName C 2024-11-14T02:45:08,184 DEBUG [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:08,185 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.HStore(327): Store=ae5c683ab910cad12ebb3cb2593f891b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:08,185 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:08,185 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:08,185 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:08,186 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:08,186 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:08,186 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:08,187 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:45:08,188 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:08,189 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:45:08,189 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened ae5c683ab910cad12ebb3cb2593f891b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68683078, jitterRate=0.023457616567611694}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:45:08,189 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:08,190 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for ae5c683ab910cad12ebb3cb2593f891b: Running coprocessor pre-open hook at 1731552308179Writing region info on filesystem at 1731552308179Initializing all the Stores at 1731552308180 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552308180Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552308180Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552308180Cleaning up temporary data from old regions at 1731552308186 (+6 ms)Running coprocessor post-open hooks at 1731552308189 (+3 ms)Region opened successfully at 1731552308190 (+1 ms) 2024-11-14T02:45:08,191 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., pid=81, masterSystemTime=1731552308170 2024-11-14T02:45:08,192 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:08,192 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:08,193 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=ae5c683ab910cad12ebb3cb2593f891b, regionState=OPEN, openSeqNum=2, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:08,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:45:08,197 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-14T02:45:08,197 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 in 177 msec 2024-11-14T02:45:08,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-14T02:45:08,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b, ASSIGN in 337 msec 2024-11-14T02:45:08,200 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T02:45:08,200 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552308200"}]},"ts":"1731552308200"} 2024-11-14T02:45:08,201 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T02:45:08,202 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T02:45:08,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1770 sec 2024-11-14T02:45:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-14T02:45:09,187 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:09,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b5885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:09,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:09,190 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:09,192 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:09,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c67def4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:09,195 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e2d6940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:09,196 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:09,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:09,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:09,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:09,200 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:45:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:45:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7adfb8b3 2024-11-14T02:45:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:45:09,201 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:45:09,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:45:09,202 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:45:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:09,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742138_1314 (size=1097) 2024-11-14T02:45:09,617 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T02:45:09,617 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T02:45:09,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:45:09,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b, REOPEN/MOVE}] 2024-11-14T02:45:09,690 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b, REOPEN/MOVE 2024-11-14T02:45:09,691 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=ae5c683ab910cad12ebb3cb2593f891b, regionState=CLOSING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:09,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b, REOPEN/MOVE because future has completed 2024-11-14T02:45:09,692 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:45:09,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:45:09,845 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:09,845 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:45:09,845 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing ae5c683ab910cad12ebb3cb2593f891b, disabling compactions & flushes 2024-11-14T02:45:09,845 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:09,845 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:09,845 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. after waiting 0 ms 2024-11-14T02:45:09,845 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:09,852 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T02:45:09,853 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:09,853 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for ae5c683ab910cad12ebb3cb2593f891b: Waiting for close lock at 1731552309845Running coprocessor pre-close hooks at 1731552309845Disabling compacts and flushes for region at 1731552309845Disabling writes for close at 1731552309845Writing region close event to WAL at 1731552309846 (+1 ms)Running coprocessor post-close hooks at 1731552309853 (+7 ms)Closed at 1731552309853 2024-11-14T02:45:09,853 WARN [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: ae5c683ab910cad12ebb3cb2593f891b to self. 2024-11-14T02:45:09,856 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:09,857 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=ae5c683ab910cad12ebb3cb2593f891b, regionState=CLOSED 2024-11-14T02:45:09,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:45:09,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-14T02:45:09,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 in 168 msec 2024-11-14T02:45:09,863 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b, REOPEN/MOVE; state=CLOSED, location=e5dc8b5cf5e9,46387,1731552206828; forceNewPlan=false, retain=true 2024-11-14T02:45:10,014 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=ae5c683ab910cad12ebb3cb2593f891b, regionState=OPENING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b, REOPEN/MOVE because future has completed 2024-11-14T02:45:10,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:45:10,177 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:10,177 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7752): Opening region: {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:45:10,178 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:10,178 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:45:10,178 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7794): checking encryption for ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:10,178 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7797): checking classloading for ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:10,180 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:10,181 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:45:10,181 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae5c683ab910cad12ebb3cb2593f891b columnFamilyName A 2024-11-14T02:45:10,183 DEBUG [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:10,184 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.HStore(327): Store=ae5c683ab910cad12ebb3cb2593f891b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:10,184 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:10,185 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:45:10,185 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae5c683ab910cad12ebb3cb2593f891b columnFamilyName B 2024-11-14T02:45:10,185 DEBUG [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:10,186 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.HStore(327): Store=ae5c683ab910cad12ebb3cb2593f891b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:10,186 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:10,187 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:45:10,187 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae5c683ab910cad12ebb3cb2593f891b columnFamilyName C 2024-11-14T02:45:10,187 DEBUG [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:10,187 INFO [StoreOpener-ae5c683ab910cad12ebb3cb2593f891b-1 {}] regionserver.HStore(327): Store=ae5c683ab910cad12ebb3cb2593f891b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:10,188 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:10,188 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1038): replaying wal for ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:10,188 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:10,190 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:10,191 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1048): stopping wal replay for ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:10,191 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1060): Cleaning up temporary data for ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:10,191 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:45:10,193 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1093): writing seq id for ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:10,194 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1114): Opened ae5c683ab910cad12ebb3cb2593f891b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58745834, jitterRate=-0.12461885809898376}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:45:10,194 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:10,195 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1006): Region open journal for ae5c683ab910cad12ebb3cb2593f891b: Running coprocessor pre-open hook at 1731552310178Writing region info on filesystem at 1731552310178Initializing all the Stores at 1731552310179 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552310179Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552310180 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552310180Cleaning up temporary data from old regions at 1731552310191 (+11 ms)Running coprocessor post-open hooks at 1731552310194 (+3 ms)Region opened successfully at 1731552310195 (+1 ms) 2024-11-14T02:45:10,196 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., pid=86, masterSystemTime=1731552310169 2024-11-14T02:45:10,198 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:10,198 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:10,198 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=ae5c683ab910cad12ebb3cb2593f891b, regionState=OPEN, openSeqNum=5, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,199 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38717 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=e5dc8b5cf5e9,46387,1731552206828, table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-14T02:45:10,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:45:10,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=84 2024-11-14T02:45:10,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 in 185 msec 2024-11-14T02:45:10,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-14T02:45:10,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b, REOPEN/MOVE in 515 msec 2024-11-14T02:45:10,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-14T02:45:10,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 521 msec 2024-11-14T02:45:10,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0050 sec 2024-11-14T02:45:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T02:45:10,210 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:10,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,210 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:10,211 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:10,211 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@191d2b46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:10,212 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:10,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6439980e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,213 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:10,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:10,214 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48d93cd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:10,215 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3744a99f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:10,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:10,217 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cd811f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:10,218 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:10,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@522a26d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,219 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:10,219 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:10,219 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:10,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e99f17c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:10,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:10,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:10,220 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:10,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54719020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:10,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:10,222 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48144ef9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:10,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:10,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:10,223 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:10,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@499d44ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:10,225 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:10,225 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eae92e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:10,226 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f67e4ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,227 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:10,227 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:10,228 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:10,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:10,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:10,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@369c613d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:10,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:10,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:10,229 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d52d3e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,230 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:10,230 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:10,231 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51e78fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:10,232 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f3e72e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,233 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:10,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:10,234 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5acc1995, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:10,235 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:10,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d3767b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b779047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,236 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:10,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:10,237 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:10,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ac6656, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:10,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:10,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:10,239 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:10,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6849674, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:10,239 DEBUG [Thread-1453 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:10,240 DEBUG [Thread-1464 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:10,240 DEBUG [Thread-1466 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:10,240 DEBUG [Thread-1455 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:10,240 DEBUG [Thread-1458 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:10,240 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-14T02:45:10,242 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:10,242 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:10,242 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:10,242 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:10,242 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:10,243 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:10,243 DEBUG [Thread-1462 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T02:45:10,243 DEBUG [Thread-1451 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:10,243 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:10,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:10,243 DEBUG [Thread-1447 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:10,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:10,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:10,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:10,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:10,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:45:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:10,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:45:10,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:45:10,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:10,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:10,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:45:10,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:45:10,248 DEBUG [Thread-1460 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:10,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:45:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:45:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:45:10,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:45:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:10,256 DEBUG [Thread-1449 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:10,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae5c683ab910cad12ebb3cb2593f891b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:45:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=A 2024-11-14T02:45:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=B 2024-11-14T02:45:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=C 2024-11-14T02:45:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:10,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:45:10,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fd3e8710bb6745b5b995aec1e0cacb04_ae5c683ab910cad12ebb3cb2593f891b is 50, key is test_row_0/A:col10/1731552310247/Put/seqid=0 2024-11-14T02:45:10,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731552310268, max timestamp=1731552310292Num uniques 859; 2024-11-14T02:45:10,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731552310268, max timestamp=1731552310292Num uniques 859; 2024-11-14T02:45:10,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T02:45:10,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T02:45:10,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731552310268, max timestamp=1731552310292Num uniques 859; 2024-11-14T02:45:10,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T02:45:10,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742139_1315 (size=12154) 2024-11-14T02:45:10,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552370322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552370323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552370326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552370326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552370327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T02:45:10,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T02:45:10,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:10,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:10,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:10,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:10,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:10,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552370435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552370445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552370445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552370445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552370445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T02:45:10,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:10,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:10,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:10,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:10,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T02:45:10,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552370655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552370665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552370665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552370665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552370665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T02:45:10,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:10,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:10,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:10,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:10,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:10,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:10,703 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:10,706 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fd3e8710bb6745b5b995aec1e0cacb04_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fd3e8710bb6745b5b995aec1e0cacb04_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:10,707 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/c85aa69e38b14b98b785291cfccb1ff8, store: [table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:10,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/c85aa69e38b14b98b785291cfccb1ff8 is 175, key is test_row_0/A:col10/1731552310247/Put/seqid=0 2024-11-14T02:45:10,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742140_1316 (size=30955) 2024-11-14T02:45:10,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T02:45:10,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:10,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:10,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:10,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:10,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T02:45:10,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552370965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552370975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552370975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552370975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:10,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:10,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552370985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:11,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T02:45:11,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:11,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:11,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:11,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,111 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/c85aa69e38b14b98b785291cfccb1ff8 2024-11-14T02:45:11,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/476235a588d04f629f5a108ba6a04b84 is 50, key is test_row_0/B:col10/1731552310247/Put/seqid=0 2024-11-14T02:45:11,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742141_1317 (size=12001) 2024-11-14T02:45:11,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T02:45:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:11,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T02:45:11,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:11,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:11,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:11,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T02:45:11,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T02:45:11,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:11,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:11,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:11,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552371485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:11,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552371495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:11,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552371496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:11,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552371496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:11,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552371496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:11,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/476235a588d04f629f5a108ba6a04b84 2024-11-14T02:45:11,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/c6a85f0d24a149f9a9816e494284c314 is 50, key is test_row_0/C:col10/1731552310247/Put/seqid=0 2024-11-14T02:45:11,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742142_1318 (size=12001) 2024-11-14T02:45:11,606 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T02:45:11,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T02:45:11,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:11,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:11,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:11,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T02:45:11,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:11,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:11,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:11,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T02:45:11,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:11,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:11,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:11,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:11,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/c6a85f0d24a149f9a9816e494284c314 2024-11-14T02:45:11,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/c85aa69e38b14b98b785291cfccb1ff8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/c85aa69e38b14b98b785291cfccb1ff8 2024-11-14T02:45:11,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/c85aa69e38b14b98b785291cfccb1ff8, entries=150, sequenceid=14, filesize=30.2 K 2024-11-14T02:45:11,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/476235a588d04f629f5a108ba6a04b84 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/476235a588d04f629f5a108ba6a04b84 2024-11-14T02:45:11,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/476235a588d04f629f5a108ba6a04b84, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T02:45:11,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/c6a85f0d24a149f9a9816e494284c314 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c6a85f0d24a149f9a9816e494284c314 2024-11-14T02:45:11,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c6a85f0d24a149f9a9816e494284c314, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T02:45:11,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ae5c683ab910cad12ebb3cb2593f891b in 1722ms, sequenceid=14, compaction requested=false 2024-11-14T02:45:11,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:12,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T02:45:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:12,077 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing ae5c683ab910cad12ebb3cb2593f891b 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-14T02:45:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=A 2024-11-14T02:45:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=B 2024-11-14T02:45:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=C 2024-11-14T02:45:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b00a2381aaec4198a6b2eef0bf7bcfeb_ae5c683ab910cad12ebb3cb2593f891b is 50, key is test_row_0/A:col10/1731552310310/Put/seqid=0 2024-11-14T02:45:12,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742143_1319 (size=12154) 2024-11-14T02:45:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T02:45:12,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:12,499 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b00a2381aaec4198a6b2eef0bf7bcfeb_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b00a2381aaec4198a6b2eef0bf7bcfeb_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:12,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/d344b6bf6555443b8c4a2c56b7502a86, store: [table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:12,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/d344b6bf6555443b8c4a2c56b7502a86 is 175, key is test_row_0/A:col10/1731552310310/Put/seqid=0 2024-11-14T02:45:12,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742144_1320 (size=30955) 2024-11-14T02:45:12,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:12,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552372526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:12,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:12,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552372526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:12,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552372527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:12,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552372528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:12,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552372528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:12,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552372635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:12,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552372635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:12,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552372646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:12,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:12,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552372646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:12,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:12,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:12,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552372646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:12,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:12,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552372855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:12,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552372855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:12,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552372865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:12,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552372865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:12,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552372865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:12,905 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/d344b6bf6555443b8c4a2c56b7502a86 2024-11-14T02:45:12,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/d836554cfa7c42399d0127f159b57f0b is 50, key is test_row_0/B:col10/1731552310310/Put/seqid=0 2024-11-14T02:45:12,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742145_1321 (size=12001) 2024-11-14T02:45:13,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552373175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:13,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552373175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:13,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:13,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552373185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:13,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552373185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:13,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552373185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:13,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:13,317 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/d836554cfa7c42399d0127f159b57f0b 2024-11-14T02:45:13,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/12fa951b15aa41f598f830616d1efb9f is 50, key is test_row_0/C:col10/1731552310310/Put/seqid=0 2024-11-14T02:45:13,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742146_1322 (size=12001) 2024-11-14T02:45:13,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552373695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:13,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552373695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:13,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552373695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:13,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:13,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552373705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:13,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552373705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:13,727 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/12fa951b15aa41f598f830616d1efb9f 2024-11-14T02:45:13,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/d344b6bf6555443b8c4a2c56b7502a86 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/d344b6bf6555443b8c4a2c56b7502a86 2024-11-14T02:45:13,734 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/d344b6bf6555443b8c4a2c56b7502a86, entries=150, sequenceid=48, filesize=30.2 K 2024-11-14T02:45:13,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/d836554cfa7c42399d0127f159b57f0b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d836554cfa7c42399d0127f159b57f0b 2024-11-14T02:45:13,738 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d836554cfa7c42399d0127f159b57f0b, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T02:45:13,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/12fa951b15aa41f598f830616d1efb9f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/12fa951b15aa41f598f830616d1efb9f 2024-11-14T02:45:13,742 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/12fa951b15aa41f598f830616d1efb9f, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T02:45:13,743 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=53.67 KB/54960 for ae5c683ab910cad12ebb3cb2593f891b in 1666ms, sequenceid=48, compaction requested=false 2024-11-14T02:45:13,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:13,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:13,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-14T02:45:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-14T02:45:13,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-14T02:45:13,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5010 sec 2024-11-14T02:45:13,747 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 3.5060 sec 2024-11-14T02:45:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T02:45:14,396 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:14,396 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-14T02:45:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T02:45:14,398 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:14,398 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:14,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T02:45:14,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T02:45:14,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:14,550 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing ae5c683ab910cad12ebb3cb2593f891b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T02:45:14,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=A 2024-11-14T02:45:14,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:14,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=B 2024-11-14T02:45:14,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:14,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=C 2024-11-14T02:45:14,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:14,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114027c2f6bed6e4f4dab2a9dbd727591d2_ae5c683ab910cad12ebb3cb2593f891b is 50, key is test_row_0/A:col10/1731552312526/Put/seqid=0 2024-11-14T02:45:14,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742147_1323 (size=12154) 2024-11-14T02:45:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T02:45:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:14,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:14,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552374742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:14,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552374743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:14,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731552314737, max timestamp=1731552314750Num uniques 859; 2024-11-14T02:45:14,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:45:14,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731552314737, max timestamp=1731552314750Num uniques 859; 2024-11-14T02:45:14,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731552314737, max timestamp=1731552314750Num uniques 859; 2024-11-14T02:45:14,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:45:14,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:45:14,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552374795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:14,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552374796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:14,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552374797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:14,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552374855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:14,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552374855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:14,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552374905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:14,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552374915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:14,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:14,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552374915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:14,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:14,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:14,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:14,962 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114027c2f6bed6e4f4dab2a9dbd727591d2_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114027c2f6bed6e4f4dab2a9dbd727591d2_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:14,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/c9d92d4eba684d08b0a25999b206df47, store: [table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:14,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/c9d92d4eba684d08b0a25999b206df47 is 175, key is test_row_0/A:col10/1731552312526/Put/seqid=0 2024-11-14T02:45:14,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742148_1324 (size=30955) 2024-11-14T02:45:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T02:45:15,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552375065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:15,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552375075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:15,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552375125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:15,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552375135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:15,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552375135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:15,366 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/c9d92d4eba684d08b0a25999b206df47 2024-11-14T02:45:15,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/4b30020161864461ba7630df6ee04f22 is 50, key is test_row_0/B:col10/1731552312526/Put/seqid=0 2024-11-14T02:45:15,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742149_1325 (size=12001) 2024-11-14T02:45:15,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552375376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:15,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:15,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552375395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:15,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:15,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552375435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:15,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552375445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:15,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552375455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T02:45:15,777 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/4b30020161864461ba7630df6ee04f22 2024-11-14T02:45:15,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/22dd0060bbfa415fb9ce644692b6637f is 50, key is test_row_0/C:col10/1731552312526/Put/seqid=0 2024-11-14T02:45:15,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742150_1326 (size=12001) 2024-11-14T02:45:15,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552375895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:15,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552375905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:15,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:15,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552375955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:15,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552375955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:15,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552375975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:16,187 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/22dd0060bbfa415fb9ce644692b6637f 2024-11-14T02:45:16,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/c9d92d4eba684d08b0a25999b206df47 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/c9d92d4eba684d08b0a25999b206df47 2024-11-14T02:45:16,194 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/c9d92d4eba684d08b0a25999b206df47, entries=150, sequenceid=59, filesize=30.2 K 2024-11-14T02:45:16,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/4b30020161864461ba7630df6ee04f22 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/4b30020161864461ba7630df6ee04f22 2024-11-14T02:45:16,199 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/4b30020161864461ba7630df6ee04f22, entries=150, sequenceid=59, filesize=11.7 K 2024-11-14T02:45:16,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/22dd0060bbfa415fb9ce644692b6637f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/22dd0060bbfa415fb9ce644692b6637f 2024-11-14T02:45:16,202 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/22dd0060bbfa415fb9ce644692b6637f, entries=150, sequenceid=59, filesize=11.7 K 2024-11-14T02:45:16,203 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ae5c683ab910cad12ebb3cb2593f891b in 1653ms, sequenceid=59, compaction requested=true 2024-11-14T02:45:16,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:16,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:16,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-14T02:45:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-14T02:45:16,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-14T02:45:16,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8060 sec 2024-11-14T02:45:16,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 1.8100 sec 2024-11-14T02:45:16,489 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T02:45:16,489 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T02:45:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T02:45:16,545 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:16,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-14T02:45:16,547 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T02:45:16,547 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:16,547 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T02:45:16,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T02:45:16,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:16,699 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing ae5c683ab910cad12ebb3cb2593f891b 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-14T02:45:16,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=A 2024-11-14T02:45:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=B 2024-11-14T02:45:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=C 2024-11-14T02:45:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:16,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e12cca5843184c80992cd1efa53cffd2_ae5c683ab910cad12ebb3cb2593f891b is 50, key is test_row_0/A:col10/1731552314796/Put/seqid=0 2024-11-14T02:45:16,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742151_1327 (size=12154) 2024-11-14T02:45:16,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:16,714 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e12cca5843184c80992cd1efa53cffd2_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e12cca5843184c80992cd1efa53cffd2_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:16,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/16636be1cf7f41cc9d2a940a83095a0f, store: [table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:16,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/16636be1cf7f41cc9d2a940a83095a0f is 175, key is test_row_0/A:col10/1731552314796/Put/seqid=0 2024-11-14T02:45:16,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742152_1328 (size=30955) 2024-11-14T02:45:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T02:45:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:16,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:16,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552376958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:16,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:16,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:16,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:16,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552376965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:16,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:16,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:16,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:16,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552376975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:16,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552376975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:16,986 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in ae5c683ab910cad12ebb3cb2593f891b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:45:16,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:16,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:16,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:16,986 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_2' in ae5c683ab910cad12ebb3cb2593f891b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:45:16,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:16,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:16,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:17,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552376996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:17,001 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_0' in ae5c683ab910cad12ebb3cb2593f891b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:45:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:17,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552377075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:17,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552377085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:17,121 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/16636be1cf7f41cc9d2a940a83095a0f 2024-11-14T02:45:17,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/49a6a564cfe44bf1a37c897f77cdb027 is 50, key is test_row_0/B:col10/1731552314796/Put/seqid=0 2024-11-14T02:45:17,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742153_1329 (size=12001) 2024-11-14T02:45:17,131 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/49a6a564cfe44bf1a37c897f77cdb027 2024-11-14T02:45:17,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/2dd7b467d4de461bb9fff9daba2b65b6 is 50, key is test_row_0/C:col10/1731552314796/Put/seqid=0 2024-11-14T02:45:17,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742154_1330 (size=12001) 2024-11-14T02:45:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T02:45:17,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552377295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:17,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:17,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552377305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:17,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:17,544 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/2dd7b467d4de461bb9fff9daba2b65b6 2024-11-14T02:45:17,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/16636be1cf7f41cc9d2a940a83095a0f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/16636be1cf7f41cc9d2a940a83095a0f 2024-11-14T02:45:17,552 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/16636be1cf7f41cc9d2a940a83095a0f, entries=150, sequenceid=93, filesize=30.2 K 2024-11-14T02:45:17,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/49a6a564cfe44bf1a37c897f77cdb027 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/49a6a564cfe44bf1a37c897f77cdb027 2024-11-14T02:45:17,556 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/49a6a564cfe44bf1a37c897f77cdb027, entries=150, sequenceid=93, filesize=11.7 K 2024-11-14T02:45:17,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/2dd7b467d4de461bb9fff9daba2b65b6 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/2dd7b467d4de461bb9fff9daba2b65b6 2024-11-14T02:45:17,560 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/2dd7b467d4de461bb9fff9daba2b65b6, entries=150, sequenceid=93, filesize=11.7 K 2024-11-14T02:45:17,561 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for ae5c683ab910cad12ebb3cb2593f891b in 862ms, sequenceid=93, compaction requested=true 2024-11-14T02:45:17,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:17,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:17,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-14T02:45:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-14T02:45:17,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-14T02:45:17,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0160 sec 2024-11-14T02:45:17,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.0200 sec 2024-11-14T02:45:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:17,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae5c683ab910cad12ebb3cb2593f891b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T02:45:17,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=A 2024-11-14T02:45:17,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:17,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=B 2024-11-14T02:45:17,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:17,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=C 2024-11-14T02:45:17,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:17,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114161291e09b9f465ca3fa6ccadc4b1fa1_ae5c683ab910cad12ebb3cb2593f891b is 50, key is test_row_0/A:col10/1731552317621/Put/seqid=0 2024-11-14T02:45:17,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742155_1331 (size=14594) 2024-11-14T02:45:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T02:45:17,685 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:17,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-14T02:45:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T02:45:17,687 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:17,687 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:17,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:17,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552377705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:17,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552377707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T02:45:17,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552377825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:17,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:17,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552377825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:17,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:17,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:17,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:17,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T02:45:17,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:17,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:17,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:17,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:17,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:17,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T02:45:17,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:17,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:17,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:17,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:17,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T02:45:18,034 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:18,037 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114161291e09b9f465ca3fa6ccadc4b1fa1_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114161291e09b9f465ca3fa6ccadc4b1fa1_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:18,038 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/d4c80f08e023471b9550356784cc4ccb, store: [table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:18,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/d4c80f08e023471b9550356784cc4ccb is 175, key is test_row_0/A:col10/1731552317621/Put/seqid=0 2024-11-14T02:45:18,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742156_1332 (size=39549) 2024-11-14T02:45:18,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552378045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:18,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552378045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:18,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T02:45:18,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:18,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T02:45:18,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:18,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T02:45:18,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552378365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:18,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552378365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:18,441 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/d4c80f08e023471b9550356784cc4ccb 2024-11-14T02:45:18,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/5560015b41224efb8224033d50370051 is 50, key is test_row_0/B:col10/1731552317621/Put/seqid=0 2024-11-14T02:45:18,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742157_1333 (size=12001) 2024-11-14T02:45:18,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T02:45:18,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:18,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T02:45:18,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:18,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T02:45:18,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:18,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T02:45:18,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/5560015b41224efb8224033d50370051 2024-11-14T02:45:18,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/c8b0b7b77f594ce1913e35da704004e1 is 50, key is test_row_0/C:col10/1731552317621/Put/seqid=0 2024-11-14T02:45:18,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742158_1334 (size=12001) 2024-11-14T02:45:18,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/c8b0b7b77f594ce1913e35da704004e1 2024-11-14T02:45:18,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/d4c80f08e023471b9550356784cc4ccb as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/d4c80f08e023471b9550356784cc4ccb 2024-11-14T02:45:18,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/d4c80f08e023471b9550356784cc4ccb, entries=200, sequenceid=105, filesize=38.6 K 2024-11-14T02:45:18,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/5560015b41224efb8224033d50370051 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/5560015b41224efb8224033d50370051 2024-11-14T02:45:18,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/5560015b41224efb8224033d50370051, entries=150, sequenceid=105, filesize=11.7 K 2024-11-14T02:45:18,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/c8b0b7b77f594ce1913e35da704004e1 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c8b0b7b77f594ce1913e35da704004e1 2024-11-14T02:45:18,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c8b0b7b77f594ce1913e35da704004e1, entries=150, sequenceid=105, filesize=11.7 K 2024-11-14T02:45:18,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ae5c683ab910cad12ebb3cb2593f891b in 1257ms, sequenceid=105, compaction requested=true 2024-11-14T02:45:18,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:18,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae5c683ab910cad12ebb3cb2593f891b:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:18,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:18,880 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T02:45:18,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae5c683ab910cad12ebb3cb2593f891b:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:18,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:18,880 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T02:45:18,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae5c683ab910cad12ebb3cb2593f891b:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:18,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:18,881 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 163369 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T02:45:18,881 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T02:45:18,881 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ae5c683ab910cad12ebb3cb2593f891b/A is initiating minor compaction (all files) 2024-11-14T02:45:18,881 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ae5c683ab910cad12ebb3cb2593f891b/B is initiating minor compaction (all files) 2024-11-14T02:45:18,881 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae5c683ab910cad12ebb3cb2593f891b/A in TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,881 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae5c683ab910cad12ebb3cb2593f891b/B in TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,881 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/c85aa69e38b14b98b785291cfccb1ff8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/d344b6bf6555443b8c4a2c56b7502a86, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/c9d92d4eba684d08b0a25999b206df47, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/16636be1cf7f41cc9d2a940a83095a0f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/d4c80f08e023471b9550356784cc4ccb] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp, totalSize=159.5 K 2024-11-14T02:45:18,881 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/476235a588d04f629f5a108ba6a04b84, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d836554cfa7c42399d0127f159b57f0b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/4b30020161864461ba7630df6ee04f22, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/49a6a564cfe44bf1a37c897f77cdb027, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/5560015b41224efb8224033d50370051] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp, totalSize=58.6 K 2024-11-14T02:45:18,881 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,881 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/c85aa69e38b14b98b785291cfccb1ff8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/d344b6bf6555443b8c4a2c56b7502a86, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/c9d92d4eba684d08b0a25999b206df47, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/16636be1cf7f41cc9d2a940a83095a0f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/d4c80f08e023471b9550356784cc4ccb] 2024-11-14T02:45:18,882 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 476235a588d04f629f5a108ba6a04b84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731552310247 2024-11-14T02:45:18,882 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting c85aa69e38b14b98b785291cfccb1ff8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731552310247 2024-11-14T02:45:18,882 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting d836554cfa7c42399d0127f159b57f0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731552310310 2024-11-14T02:45:18,882 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting d344b6bf6555443b8c4a2c56b7502a86, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731552310310 2024-11-14T02:45:18,882 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b30020161864461ba7630df6ee04f22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552312524 2024-11-14T02:45:18,882 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9d92d4eba684d08b0a25999b206df47, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552312524 2024-11-14T02:45:18,882 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 49a6a564cfe44bf1a37c897f77cdb027, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731552314794 2024-11-14T02:45:18,883 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16636be1cf7f41cc9d2a940a83095a0f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731552314794 2024-11-14T02:45:18,883 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 5560015b41224efb8224033d50370051, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731552316937 2024-11-14T02:45:18,883 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4c80f08e023471b9550356784cc4ccb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731552316937 2024-11-14T02:45:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:18,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552317692, max timestamp=1731552318893Num uniques 859; 2024-11-14T02:45:18,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552317692, max timestamp=1731552318893Num uniques 859; 2024-11-14T02:45:18,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:45:18,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:45:18,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552317692, max timestamp=1731552318893Num uniques 859; 2024-11-14T02:45:18,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:45:18,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae5c683ab910cad12ebb3cb2593f891b 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-14T02:45:18,897 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:18,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=A 2024-11-14T02:45:18,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:18,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=B 2024-11-14T02:45:18,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:18,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=C 2024-11-14T02:45:18,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:18,899 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae5c683ab910cad12ebb3cb2593f891b#B#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:45:18,900 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/0093c8a9b43c404ca14c0a3791c23d90 is 50, key is test_row_0/B:col10/1731552317621/Put/seqid=0 2024-11-14T02:45:18,905 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111469661040899a4baeb3b45b81865f1881_ae5c683ab910cad12ebb3cb2593f891b store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:18,909 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111469661040899a4baeb3b45b81865f1881_ae5c683ab910cad12ebb3cb2593f891b, store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:18,910 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111469661040899a4baeb3b45b81865f1881_ae5c683ab910cad12ebb3cb2593f891b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:18,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142e844b9e3f4141cb8caf54bbc4abc125_ae5c683ab910cad12ebb3cb2593f891b is 50, key is test_row_0/A:col10/1731552317692/Put/seqid=0 2024-11-14T02:45:18,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T02:45:18,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:18,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:18,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742159_1335 (size=12173) 2024-11-14T02:45:18,927 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/0093c8a9b43c404ca14c0a3791c23d90 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/0093c8a9b43c404ca14c0a3791c23d90 2024-11-14T02:45:18,931 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ae5c683ab910cad12ebb3cb2593f891b/B of ae5c683ab910cad12ebb3cb2593f891b into 0093c8a9b43c404ca14c0a3791c23d90(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:18,931 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:18,931 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., storeName=ae5c683ab910cad12ebb3cb2593f891b/B, priority=11, startTime=1731552318880; duration=0sec 2024-11-14T02:45:18,931 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:18,931 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae5c683ab910cad12ebb3cb2593f891b:B 2024-11-14T02:45:18,931 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T02:45:18,935 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T02:45:18,935 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ae5c683ab910cad12ebb3cb2593f891b/C is initiating minor compaction (all files) 2024-11-14T02:45:18,936 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae5c683ab910cad12ebb3cb2593f891b/C in TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:18,936 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c6a85f0d24a149f9a9816e494284c314, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/12fa951b15aa41f598f830616d1efb9f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/22dd0060bbfa415fb9ce644692b6637f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/2dd7b467d4de461bb9fff9daba2b65b6, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c8b0b7b77f594ce1913e35da704004e1] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp, totalSize=58.6 K 2024-11-14T02:45:18,936 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting c6a85f0d24a149f9a9816e494284c314, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731552310247 2024-11-14T02:45:18,936 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 12fa951b15aa41f598f830616d1efb9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731552310310 2024-11-14T02:45:18,937 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 22dd0060bbfa415fb9ce644692b6637f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552312524 2024-11-14T02:45:18,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742160_1336 (size=4469) 2024-11-14T02:45:18,937 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dd7b467d4de461bb9fff9daba2b65b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731552314794 2024-11-14T02:45:18,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742161_1337 (size=14594) 2024-11-14T02:45:18,938 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting c8b0b7b77f594ce1913e35da704004e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731552316937 2024-11-14T02:45:18,938 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:18,942 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142e844b9e3f4141cb8caf54bbc4abc125_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142e844b9e3f4141cb8caf54bbc4abc125_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:18,943 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/9f341e6af414436f9889d1ef1046f7d8, store: [table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:18,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/9f341e6af414436f9889d1ef1046f7d8 is 175, key is test_row_0/A:col10/1731552317692/Put/seqid=0 2024-11-14T02:45:18,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742162_1338 (size=39549) 2024-11-14T02:45:18,948 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/9f341e6af414436f9889d1ef1046f7d8 2024-11-14T02:45:18,952 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae5c683ab910cad12ebb3cb2593f891b#C#compaction#287 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:45:18,952 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/320061ac74f349afa50432f0d666fdc8 is 50, key is test_row_0/C:col10/1731552317621/Put/seqid=0 2024-11-14T02:45:18,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/975cd801d260412ea962659f95bb9ad0 is 50, key is test_row_0/B:col10/1731552317692/Put/seqid=0 2024-11-14T02:45:18,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742164_1340 (size=12001) 2024-11-14T02:45:18,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742163_1339 (size=12173) 2024-11-14T02:45:19,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552318978, max timestamp=1731552318999Num uniques 859; 2024-11-14T02:45:19,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:45:19,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552318978, max timestamp=1731552318999Num uniques 859; 2024-11-14T02:45:19,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:45:19,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552318978, max timestamp=1731552318999Num uniques 859; 2024-11-14T02:45:19,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:45:19,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T02:45:19,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:19,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:19,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:19,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731552319036, max timestamp=1731552319062Num uniques 1018; 2024-11-14T02:45:19,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=16848, suffix heap size=133840,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=368 2024-11-14T02:45:19,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731552319036, max timestamp=1731552319062Num uniques 1018; 2024-11-14T02:45:19,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=16848, suffix heap size=133840,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=368 2024-11-14T02:45:19,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731552319036, max timestamp=1731552319062Num uniques 1018; 2024-11-14T02:45:19,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=16848, suffix heap size=133840,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=368 2024-11-14T02:45:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552379100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552379104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552379105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552379115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552379116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T02:45:19,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:19,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:19,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:19,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552379225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552379225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552379225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552379225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552379245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,338 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae5c683ab910cad12ebb3cb2593f891b#A#compaction#285 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:45:19,339 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/5fca256a29e2461b8f20755dbddd2820 is 175, key is test_row_0/A:col10/1731552317621/Put/seqid=0 2024-11-14T02:45:19,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742165_1341 (size=31127) 2024-11-14T02:45:19,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/975cd801d260412ea962659f95bb9ad0 2024-11-14T02:45:19,370 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/320061ac74f349afa50432f0d666fdc8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/320061ac74f349afa50432f0d666fdc8 2024-11-14T02:45:19,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T02:45:19,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:19,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:19,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:19,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/57b2b809b66d4a748166c0575a0c32c7 is 50, key is test_row_0/C:col10/1731552317692/Put/seqid=0 2024-11-14T02:45:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,378 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ae5c683ab910cad12ebb3cb2593f891b/C of ae5c683ab910cad12ebb3cb2593f891b into 320061ac74f349afa50432f0d666fdc8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:19,378 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:19,378 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., storeName=ae5c683ab910cad12ebb3cb2593f891b/C, priority=11, startTime=1731552318880; duration=0sec 2024-11-14T02:45:19,378 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:19,378 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae5c683ab910cad12ebb3cb2593f891b:C 2024-11-14T02:45:19,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742166_1342 (size=12001) 2024-11-14T02:45:19,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552379436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552379436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552379436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552379436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552379455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T02:45:19,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:19,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:19,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:19,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T02:45:19,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:19,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:19,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:19,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552379745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552379745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552379745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552379746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552379775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:19,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/57b2b809b66d4a748166c0575a0c32c7 2024-11-14T02:45:19,827 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/5fca256a29e2461b8f20755dbddd2820 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/5fca256a29e2461b8f20755dbddd2820 2024-11-14T02:45:19,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/9f341e6af414436f9889d1ef1046f7d8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/9f341e6af414436f9889d1ef1046f7d8 2024-11-14T02:45:19,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/9f341e6af414436f9889d1ef1046f7d8, entries=200, sequenceid=127, filesize=38.6 K 2024-11-14T02:45:19,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T02:45:19,831 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ae5c683ab910cad12ebb3cb2593f891b/A of ae5c683ab910cad12ebb3cb2593f891b into 5fca256a29e2461b8f20755dbddd2820(size=30.4 K), total size for store is 69.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:19,831 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:19,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:19,831 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., storeName=ae5c683ab910cad12ebb3cb2593f891b/A, priority=11, startTime=1731552318880; duration=0sec 2024-11-14T02:45:19,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:19,831 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:19,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:19,831 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae5c683ab910cad12ebb3cb2593f891b:A 2024-11-14T02:45:19,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/975cd801d260412ea962659f95bb9ad0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/975cd801d260412ea962659f95bb9ad0 2024-11-14T02:45:19,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T02:45:19,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/975cd801d260412ea962659f95bb9ad0, entries=150, sequenceid=127, filesize=11.7 K 2024-11-14T02:45:19,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/57b2b809b66d4a748166c0575a0c32c7 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/57b2b809b66d4a748166c0575a0c32c7 2024-11-14T02:45:19,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/57b2b809b66d4a748166c0575a0c32c7, entries=150, sequenceid=127, filesize=11.7 K 2024-11-14T02:45:19,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=161.02 KB/164880 for ae5c683ab910cad12ebb3cb2593f891b in 944ms, sequenceid=127, compaction requested=false 2024-11-14T02:45:19,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:19,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T02:45:19,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:19,984 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing ae5c683ab910cad12ebb3cb2593f891b 3/3 column families, dataSize=161.02 KB heapSize=445.08 KB 2024-11-14T02:45:19,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=A 2024-11-14T02:45:19,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:19,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=B 2024-11-14T02:45:19,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:19,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=C 2024-11-14T02:45:19,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:19,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111461261a7f3ea7410d9eeebb24e4ea8580_ae5c683ab910cad12ebb3cb2593f891b is 50, key is test_row_0/A:col10/1731552319114/Put/seqid=0 2024-11-14T02:45:19,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742167_1343 (size=12304) 2024-11-14T02:45:20,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:20,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:20,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552380271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552380272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552380273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552380274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552380296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552380385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552380385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:20,401 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111461261a7f3ea7410d9eeebb24e4ea8580_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111461261a7f3ea7410d9eeebb24e4ea8580_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:20,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/631fc88c934448959a0caf520ec23c2c, store: [table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:20,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/631fc88c934448959a0caf520ec23c2c is 175, key is test_row_0/A:col10/1731552319114/Put/seqid=0 2024-11-14T02:45:20,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552380395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742168_1344 (size=31105) 2024-11-14T02:45:20,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552380395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552380605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552380605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552380615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552380615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,805 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/631fc88c934448959a0caf520ec23c2c 2024-11-14T02:45:20,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/37a2faba81fb4c09a26fa3b894831670 is 50, key is test_row_0/B:col10/1731552319114/Put/seqid=0 2024-11-14T02:45:20,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742169_1345 (size=12151) 2024-11-14T02:45:20,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552380915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552380925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552380926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:20,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552380935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,216 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/37a2faba81fb4c09a26fa3b894831670 2024-11-14T02:45:21,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/dbd175d10f964151a5b5301a52601554 is 50, key is test_row_0/C:col10/1731552319114/Put/seqid=0 2024-11-14T02:45:21,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742170_1346 (size=12151) 2024-11-14T02:45:21,231 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/dbd175d10f964151a5b5301a52601554 2024-11-14T02:45:21,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/631fc88c934448959a0caf520ec23c2c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/631fc88c934448959a0caf520ec23c2c 2024-11-14T02:45:21,237 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/631fc88c934448959a0caf520ec23c2c, entries=150, sequenceid=184, filesize=30.4 K 2024-11-14T02:45:21,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/37a2faba81fb4c09a26fa3b894831670 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/37a2faba81fb4c09a26fa3b894831670 2024-11-14T02:45:21,240 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/37a2faba81fb4c09a26fa3b894831670, entries=150, sequenceid=184, filesize=11.9 K 2024-11-14T02:45:21,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/dbd175d10f964151a5b5301a52601554 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/dbd175d10f964151a5b5301a52601554 2024-11-14T02:45:21,244 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/dbd175d10f964151a5b5301a52601554, entries=150, sequenceid=184, filesize=11.9 K 2024-11-14T02:45:21,245 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.03 KB/455712, currentSize=26.84 KB/27480 for ae5c683ab910cad12ebb3cb2593f891b in 1261ms, sequenceid=184, compaction requested=true 2024-11-14T02:45:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-14T02:45:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-14T02:45:21,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-14T02:45:21,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5590 sec 2024-11-14T02:45:21,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 3.5640 sec 2024-11-14T02:45:21,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:21,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae5c683ab910cad12ebb3cb2593f891b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:45:21,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=A 2024-11-14T02:45:21,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:21,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=B 2024-11-14T02:45:21,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:21,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=C 2024-11-14T02:45:21,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:21,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114071a6e353d2f4047936a91dd048b48e6_ae5c683ab910cad12ebb3cb2593f891b is 50, key is test_row_0/A:col10/1731552321313/Put/seqid=0 2024-11-14T02:45:21,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742171_1347 (size=14794) 2024-11-14T02:45:21,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552321389, max timestamp=1731552321441Num uniques 859; 2024-11-14T02:45:21,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:45:21,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552321389, max timestamp=1731552321441Num uniques 859; 2024-11-14T02:45:21,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:45:21,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552321389, max timestamp=1731552321441Num uniques 859; 2024-11-14T02:45:21,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:45:21,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552381518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:21,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552381519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552381520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552381532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552381536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552381645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:21,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552381645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552381645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552381645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552381655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,729 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:21,731 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114071a6e353d2f4047936a91dd048b48e6_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114071a6e353d2f4047936a91dd048b48e6_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:21,732 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/58d3cbf0d05b4ea28013473647ec2b5b, store: [table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:21,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/58d3cbf0d05b4ea28013473647ec2b5b is 175, key is test_row_0/A:col10/1731552321313/Put/seqid=0 2024-11-14T02:45:21,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742172_1348 (size=39749) 2024-11-14T02:45:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T02:45:21,855 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:21,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-14T02:45:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T02:45:21,857 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:21,858 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:21,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:21,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552381855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552381865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:21,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552381865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552381865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552381874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T02:45:22,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T02:45:22,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:22,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,136 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/58d3cbf0d05b4ea28013473647ec2b5b 2024-11-14T02:45:22,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/30d21d99d4d6453792ce7d06c88d5f71 is 50, key is test_row_0/B:col10/1731552321313/Put/seqid=0 2024-11-14T02:45:22,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742173_1349 (size=12151) 2024-11-14T02:45:22,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T02:45:22,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:22,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T02:45:22,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552382175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:22,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552382185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:22,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552382185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:22,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552382185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552382186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:22,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T02:45:22,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:22,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T02:45:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T02:45:22,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/30d21d99d4d6453792ce7d06c88d5f71 2024-11-14T02:45:22,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/deaa7eb2c30748879514b47554cd3020 is 50, key is test_row_0/C:col10/1731552321313/Put/seqid=0 2024-11-14T02:45:22,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742174_1350 (size=12151) 2024-11-14T02:45:22,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T02:45:22,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:22,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552382695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:22,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:22,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552382705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:22,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552382705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:22,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552382705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552382705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:22,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T02:45:22,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:22,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T02:45:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:22,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/deaa7eb2c30748879514b47554cd3020 2024-11-14T02:45:22,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/58d3cbf0d05b4ea28013473647ec2b5b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/58d3cbf0d05b4ea28013473647ec2b5b 2024-11-14T02:45:22,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/58d3cbf0d05b4ea28013473647ec2b5b, entries=200, sequenceid=194, filesize=38.8 K 2024-11-14T02:45:22,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/30d21d99d4d6453792ce7d06c88d5f71 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/30d21d99d4d6453792ce7d06c88d5f71 2024-11-14T02:45:22,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/30d21d99d4d6453792ce7d06c88d5f71, entries=150, sequenceid=194, filesize=11.9 K 2024-11-14T02:45:22,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/deaa7eb2c30748879514b47554cd3020 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/deaa7eb2c30748879514b47554cd3020 2024-11-14T02:45:22,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/deaa7eb2c30748879514b47554cd3020, entries=150, sequenceid=194, filesize=11.9 K 2024-11-14T02:45:22,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ae5c683ab910cad12ebb3cb2593f891b in 1648ms, sequenceid=194, compaction requested=true 2024-11-14T02:45:22,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:22,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae5c683ab910cad12ebb3cb2593f891b:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:22,967 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:45:22,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:22,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae5c683ab910cad12ebb3cb2593f891b:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:22,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:22,967 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:45:22,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae5c683ab910cad12ebb3cb2593f891b:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:22,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:22,968 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:45:22,968 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141530 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:45:22,968 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ae5c683ab910cad12ebb3cb2593f891b/B is initiating minor compaction (all files) 2024-11-14T02:45:22,968 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ae5c683ab910cad12ebb3cb2593f891b/A is initiating minor compaction (all files) 2024-11-14T02:45:22,968 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae5c683ab910cad12ebb3cb2593f891b/B in TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,968 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae5c683ab910cad12ebb3cb2593f891b/A in TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,968 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/0093c8a9b43c404ca14c0a3791c23d90, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/975cd801d260412ea962659f95bb9ad0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/37a2faba81fb4c09a26fa3b894831670, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/30d21d99d4d6453792ce7d06c88d5f71] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp, totalSize=47.3 K 2024-11-14T02:45:22,968 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/5fca256a29e2461b8f20755dbddd2820, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/9f341e6af414436f9889d1ef1046f7d8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/631fc88c934448959a0caf520ec23c2c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/58d3cbf0d05b4ea28013473647ec2b5b] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp, totalSize=138.2 K 2024-11-14T02:45:22,968 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:22,968 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/5fca256a29e2461b8f20755dbddd2820, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/9f341e6af414436f9889d1ef1046f7d8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/631fc88c934448959a0caf520ec23c2c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/58d3cbf0d05b4ea28013473647ec2b5b] 2024-11-14T02:45:22,968 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 0093c8a9b43c404ca14c0a3791c23d90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731552316937 2024-11-14T02:45:22,969 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fca256a29e2461b8f20755dbddd2820, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731552316937 2024-11-14T02:45:22,969 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 975cd801d260412ea962659f95bb9ad0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731552317692 2024-11-14T02:45:22,969 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f341e6af414436f9889d1ef1046f7d8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731552317692 2024-11-14T02:45:22,969 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 37a2faba81fb4c09a26fa3b894831670, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731552319103 2024-11-14T02:45:22,969 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 631fc88c934448959a0caf520ec23c2c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731552319103 2024-11-14T02:45:22,969 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 30d21d99d4d6453792ce7d06c88d5f71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731552321308 2024-11-14T02:45:22,969 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58d3cbf0d05b4ea28013473647ec2b5b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731552320272 2024-11-14T02:45:22,979 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:22,981 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae5c683ab910cad12ebb3cb2593f891b#B#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:45:22,981 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/d7efd121beb640b5a9de9663740069c4 is 50, key is test_row_0/B:col10/1731552321313/Put/seqid=0 2024-11-14T02:45:22,981 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411145c31854f7f734294a660606874c42488_ae5c683ab910cad12ebb3cb2593f891b store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:22,984 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411145c31854f7f734294a660606874c42488_ae5c683ab910cad12ebb3cb2593f891b, store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:22,984 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145c31854f7f734294a660606874c42488_ae5c683ab910cad12ebb3cb2593f891b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:22,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742175_1351 (size=12459) 2024-11-14T02:45:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T02:45:22,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742176_1352 (size=4469) 2024-11-14T02:45:23,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T02:45:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:23,078 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing ae5c683ab910cad12ebb3cb2593f891b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-14T02:45:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=A 2024-11-14T02:45:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=B 2024-11-14T02:45:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=C 2024-11-14T02:45:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:23,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111453c6911bb6b54716acf3cbcf389056ac_ae5c683ab910cad12ebb3cb2593f891b is 50, key is test_row_0/A:col10/1731552321535/Put/seqid=0 2024-11-14T02:45:23,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742177_1353 (size=12304) 2024-11-14T02:45:23,394 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/d7efd121beb640b5a9de9663740069c4 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d7efd121beb640b5a9de9663740069c4 2024-11-14T02:45:23,397 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae5c683ab910cad12ebb3cb2593f891b/B of ae5c683ab910cad12ebb3cb2593f891b into d7efd121beb640b5a9de9663740069c4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:23,397 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:23,397 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., storeName=ae5c683ab910cad12ebb3cb2593f891b/B, priority=12, startTime=1731552322967; duration=0sec 2024-11-14T02:45:23,397 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:23,397 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae5c683ab910cad12ebb3cb2593f891b:B 2024-11-14T02:45:23,397 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:45:23,399 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:45:23,399 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ae5c683ab910cad12ebb3cb2593f891b/C is initiating minor compaction (all files) 2024-11-14T02:45:23,399 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae5c683ab910cad12ebb3cb2593f891b/C in TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:23,399 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/320061ac74f349afa50432f0d666fdc8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/57b2b809b66d4a748166c0575a0c32c7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/dbd175d10f964151a5b5301a52601554, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/deaa7eb2c30748879514b47554cd3020] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp, totalSize=47.3 K 2024-11-14T02:45:23,399 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae5c683ab910cad12ebb3cb2593f891b#A#compaction#297 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:45:23,400 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 320061ac74f349afa50432f0d666fdc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731552316937 2024-11-14T02:45:23,400 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/a3b03e616c8a48deb3d3cf196e106b4c is 175, key is test_row_0/A:col10/1731552321313/Put/seqid=0 2024-11-14T02:45:23,400 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 57b2b809b66d4a748166c0575a0c32c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731552317692 2024-11-14T02:45:23,400 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting dbd175d10f964151a5b5301a52601554, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731552319103 2024-11-14T02:45:23,400 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting deaa7eb2c30748879514b47554cd3020, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731552321308 2024-11-14T02:45:23,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742178_1354 (size=31413) 2024-11-14T02:45:23,408 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae5c683ab910cad12ebb3cb2593f891b#C#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:45:23,409 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/1fc158116dfe46d79ee4f288c6ad89bb is 50, key is test_row_0/C:col10/1731552321313/Put/seqid=0 2024-11-14T02:45:23,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742179_1355 (size=12459) 2024-11-14T02:45:23,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:23,491 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111453c6911bb6b54716acf3cbcf389056ac_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111453c6911bb6b54716acf3cbcf389056ac_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:23,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/f84d53182820426e8dbedbdd9ba14c9c, store: [table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:23,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/f84d53182820426e8dbedbdd9ba14c9c is 175, key is test_row_0/A:col10/1731552321535/Put/seqid=0 2024-11-14T02:45:23,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742180_1356 (size=31105) 2024-11-14T02:45:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:23,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:23,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:23,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552383732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552383731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:23,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552383733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:23,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552383757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:23,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552383757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:23,807 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/a3b03e616c8a48deb3d3cf196e106b4c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/a3b03e616c8a48deb3d3cf196e106b4c 2024-11-14T02:45:23,811 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae5c683ab910cad12ebb3cb2593f891b/A of ae5c683ab910cad12ebb3cb2593f891b into a3b03e616c8a48deb3d3cf196e106b4c(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:23,811 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:23,811 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., storeName=ae5c683ab910cad12ebb3cb2593f891b/A, priority=12, startTime=1731552322967; duration=0sec 2024-11-14T02:45:23,811 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:23,811 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae5c683ab910cad12ebb3cb2593f891b:A 2024-11-14T02:45:23,817 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/1fc158116dfe46d79ee4f288c6ad89bb as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/1fc158116dfe46d79ee4f288c6ad89bb 2024-11-14T02:45:23,821 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae5c683ab910cad12ebb3cb2593f891b/C of ae5c683ab910cad12ebb3cb2593f891b into 1fc158116dfe46d79ee4f288c6ad89bb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:23,821 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:23,821 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., storeName=ae5c683ab910cad12ebb3cb2593f891b/C, priority=12, startTime=1731552322967; duration=0sec 2024-11-14T02:45:23,821 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:23,821 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae5c683ab910cad12ebb3cb2593f891b:C 2024-11-14T02:45:23,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552383865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:23,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552383865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:23,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552383865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:23,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552383875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:23,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552383875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:23,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:23,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:23,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:23,897 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/f84d53182820426e8dbedbdd9ba14c9c 2024-11-14T02:45:23,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/d456de19003043c895ba78ee733d4dce is 50, key is test_row_0/B:col10/1731552321535/Put/seqid=0 2024-11-14T02:45:23,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742181_1357 (size=12151) 2024-11-14T02:45:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T02:45:24,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552384086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:24,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552384086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:24,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552384086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552384086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552384086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:24,307 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/d456de19003043c895ba78ee733d4dce 2024-11-14T02:45:24,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/d15950aa62e247afa9e1edbdec6eed89 is 50, key is test_row_0/C:col10/1731552321535/Put/seqid=0 2024-11-14T02:45:24,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742182_1358 (size=12151) 2024-11-14T02:45:24,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552384395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552384395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552384396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:24,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552384396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:24,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:24,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552384396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:24,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:24,717 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/d15950aa62e247afa9e1edbdec6eed89 2024-11-14T02:45:24,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/f84d53182820426e8dbedbdd9ba14c9c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/f84d53182820426e8dbedbdd9ba14c9c 2024-11-14T02:45:24,724 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/f84d53182820426e8dbedbdd9ba14c9c, entries=150, sequenceid=231, filesize=30.4 K 2024-11-14T02:45:24,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/d456de19003043c895ba78ee733d4dce as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d456de19003043c895ba78ee733d4dce 2024-11-14T02:45:24,729 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d456de19003043c895ba78ee733d4dce, entries=150, sequenceid=231, filesize=11.9 K 2024-11-14T02:45:24,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/d15950aa62e247afa9e1edbdec6eed89 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/d15950aa62e247afa9e1edbdec6eed89 2024-11-14T02:45:24,733 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/d15950aa62e247afa9e1edbdec6eed89, entries=150, sequenceid=231, filesize=11.9 K 2024-11-14T02:45:24,734 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for ae5c683ab910cad12ebb3cb2593f891b in 1656ms, sequenceid=231, compaction requested=false 2024-11-14T02:45:24,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:24,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:24,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-14T02:45:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-14T02:45:24,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-14T02:45:24,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8770 sec 2024-11-14T02:45:24,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.8830 sec 2024-11-14T02:45:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:24,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae5c683ab910cad12ebb3cb2593f891b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T02:45:24,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=A 2024-11-14T02:45:24,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:24,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=B 2024-11-14T02:45:24,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:24,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=C 2024-11-14T02:45:24,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:24,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144aac2b45266f4e19a0698c5d1d46a83c_ae5c683ab910cad12ebb3cb2593f891b is 50, key is test_row_0/A:col10/1731552324919/Put/seqid=0 2024-11-14T02:45:24,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742183_1359 (size=14794) 2024-11-14T02:45:24,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552384950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:24,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552384950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:24,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731552324947, max timestamp=1731552324960Num uniques 859; 2024-11-14T02:45:24,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:45:24,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731552324947, max timestamp=1731552324960Num uniques 859; 2024-11-14T02:45:24,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:45:24,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731552324947, max timestamp=1731552324960Num uniques 859; 2024-11-14T02:45:24,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:45:25,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552385025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552385027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552385038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552385085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552385085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552385145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552385145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552385155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,195 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T02:45:25,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552385305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552385305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,329 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:25,332 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144aac2b45266f4e19a0698c5d1d46a83c_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144aac2b45266f4e19a0698c5d1d46a83c_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:25,333 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/edf63d6f7e2341af992ab2858baa6df5, store: [table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:25,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/edf63d6f7e2341af992ab2858baa6df5 is 175, key is test_row_0/A:col10/1731552324919/Put/seqid=0 2024-11-14T02:45:25,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742184_1360 (size=39749) 2024-11-14T02:45:25,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552385365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552385365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552385365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552385625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552385626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552385685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552385685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552385685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:25,738 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/edf63d6f7e2341af992ab2858baa6df5 2024-11-14T02:45:25,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/d7297a4e43c9481388998257cb7c82e5 is 50, key is test_row_0/B:col10/1731552324919/Put/seqid=0 2024-11-14T02:45:25,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742185_1361 (size=12151) 2024-11-14T02:45:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T02:45:26,025 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:26,026 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-14T02:45:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T02:45:26,027 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:26,028 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:26,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T02:45:26,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/d7297a4e43c9481388998257cb7c82e5 2024-11-14T02:45:26,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552386145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:26,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552386145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:26,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:26,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/8ef754dfb5264c1782757434b94bbbd0 is 50, key is test_row_0/C:col10/1731552324919/Put/seqid=0 2024-11-14T02:45:26,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742186_1362 (size=12151) 2024-11-14T02:45:26,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T02:45:26,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:26,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:26,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:26,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:26,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:26,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552386195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:26,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552386195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:26,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552386195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:26,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T02:45:26,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:26,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:26,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:26,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:26,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T02:45:26,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T02:45:26,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:26,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:26,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:26,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:26,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:26,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/8ef754dfb5264c1782757434b94bbbd0 2024-11-14T02:45:26,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/edf63d6f7e2341af992ab2858baa6df5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/edf63d6f7e2341af992ab2858baa6df5 2024-11-14T02:45:26,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/edf63d6f7e2341af992ab2858baa6df5, entries=200, sequenceid=245, filesize=38.8 K 2024-11-14T02:45:26,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/d7297a4e43c9481388998257cb7c82e5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d7297a4e43c9481388998257cb7c82e5 2024-11-14T02:45:26,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d7297a4e43c9481388998257cb7c82e5, entries=150, sequenceid=245, filesize=11.9 K 2024-11-14T02:45:26,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/8ef754dfb5264c1782757434b94bbbd0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/8ef754dfb5264c1782757434b94bbbd0 2024-11-14T02:45:26,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/8ef754dfb5264c1782757434b94bbbd0, entries=150, sequenceid=245, filesize=11.9 K 2024-11-14T02:45:26,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ae5c683ab910cad12ebb3cb2593f891b in 1657ms, sequenceid=245, compaction requested=true 2024-11-14T02:45:26,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:26,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae5c683ab910cad12ebb3cb2593f891b:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:26,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:26,578 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:26,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae5c683ab910cad12ebb3cb2593f891b:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:26,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:26,578 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:26,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae5c683ab910cad12ebb3cb2593f891b:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:26,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:26,579 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:26,579 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:26,579 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ae5c683ab910cad12ebb3cb2593f891b/A is initiating minor compaction (all files) 2024-11-14T02:45:26,579 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ae5c683ab910cad12ebb3cb2593f891b/B is initiating minor compaction (all files) 2024-11-14T02:45:26,579 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae5c683ab910cad12ebb3cb2593f891b/A in TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:26,579 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae5c683ab910cad12ebb3cb2593f891b/B in TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:26,579 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d7efd121beb640b5a9de9663740069c4, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d456de19003043c895ba78ee733d4dce, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d7297a4e43c9481388998257cb7c82e5] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp, totalSize=35.9 K 2024-11-14T02:45:26,579 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/a3b03e616c8a48deb3d3cf196e106b4c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/f84d53182820426e8dbedbdd9ba14c9c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/edf63d6f7e2341af992ab2858baa6df5] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp, totalSize=99.9 K 2024-11-14T02:45:26,579 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:26,579 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/a3b03e616c8a48deb3d3cf196e106b4c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/f84d53182820426e8dbedbdd9ba14c9c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/edf63d6f7e2341af992ab2858baa6df5] 2024-11-14T02:45:26,579 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting d7efd121beb640b5a9de9663740069c4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731552321308 2024-11-14T02:45:26,579 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3b03e616c8a48deb3d3cf196e106b4c, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731552321308 2024-11-14T02:45:26,580 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting d456de19003043c895ba78ee733d4dce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731552321515 2024-11-14T02:45:26,580 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting f84d53182820426e8dbedbdd9ba14c9c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731552321515 2024-11-14T02:45:26,580 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting d7297a4e43c9481388998257cb7c82e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731552323723 2024-11-14T02:45:26,580 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting edf63d6f7e2341af992ab2858baa6df5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731552323723 2024-11-14T02:45:26,587 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:26,589 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae5c683ab910cad12ebb3cb2593f891b#B#compaction#305 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:45:26,590 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/6c870ddaa7e34d2e91655c40df467566 is 50, key is test_row_0/B:col10/1731552324919/Put/seqid=0 2024-11-14T02:45:26,591 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114d72e762ad8e64449a73ccbcfce55c5bb_ae5c683ab910cad12ebb3cb2593f891b store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:26,593 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114d72e762ad8e64449a73ccbcfce55c5bb_ae5c683ab910cad12ebb3cb2593f891b, store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:26,593 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d72e762ad8e64449a73ccbcfce55c5bb_ae5c683ab910cad12ebb3cb2593f891b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:26,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742187_1363 (size=4469) 2024-11-14T02:45:26,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742188_1364 (size=12561) 2024-11-14T02:45:26,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T02:45:26,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:26,637 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing ae5c683ab910cad12ebb3cb2593f891b 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-14T02:45:26,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=A 2024-11-14T02:45:26,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:26,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=B 2024-11-14T02:45:26,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:26,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=C 2024-11-14T02:45:26,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114630ff28d34784e79b943da30b2624dfc_ae5c683ab910cad12ebb3cb2593f891b is 50, key is test_row_0/A:col10/1731552325024/Put/seqid=0 2024-11-14T02:45:26,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742189_1365 (size=12454) 2024-11-14T02:45:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T02:45:27,001 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae5c683ab910cad12ebb3cb2593f891b#A#compaction#306 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:45:27,002 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/755ed8c8aae84b5fa33fc10dca4e097e is 175, key is test_row_0/A:col10/1731552324919/Put/seqid=0 2024-11-14T02:45:27,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742190_1366 (size=31515) 2024-11-14T02:45:27,005 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/6c870ddaa7e34d2e91655c40df467566 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/6c870ddaa7e34d2e91655c40df467566 2024-11-14T02:45:27,011 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/755ed8c8aae84b5fa33fc10dca4e097e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/755ed8c8aae84b5fa33fc10dca4e097e 2024-11-14T02:45:27,013 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae5c683ab910cad12ebb3cb2593f891b/B of ae5c683ab910cad12ebb3cb2593f891b into 6c870ddaa7e34d2e91655c40df467566(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:27,013 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:27,013 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., storeName=ae5c683ab910cad12ebb3cb2593f891b/B, priority=13, startTime=1731552326578; duration=0sec 2024-11-14T02:45:27,013 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:27,013 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae5c683ab910cad12ebb3cb2593f891b:B 2024-11-14T02:45:27,013 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:27,014 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae5c683ab910cad12ebb3cb2593f891b/A of ae5c683ab910cad12ebb3cb2593f891b into 755ed8c8aae84b5fa33fc10dca4e097e(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:27,014 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:27,014 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., storeName=ae5c683ab910cad12ebb3cb2593f891b/A, priority=13, startTime=1731552326578; duration=0sec 2024-11-14T02:45:27,014 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:27,014 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae5c683ab910cad12ebb3cb2593f891b:A 2024-11-14T02:45:27,014 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:27,015 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ae5c683ab910cad12ebb3cb2593f891b/C is initiating minor compaction (all files) 2024-11-14T02:45:27,015 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae5c683ab910cad12ebb3cb2593f891b/C in TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:27,015 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/1fc158116dfe46d79ee4f288c6ad89bb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/d15950aa62e247afa9e1edbdec6eed89, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/8ef754dfb5264c1782757434b94bbbd0] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp, totalSize=35.9 K 2024-11-14T02:45:27,015 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fc158116dfe46d79ee4f288c6ad89bb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731552321308 2024-11-14T02:45:27,015 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting d15950aa62e247afa9e1edbdec6eed89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731552321515 2024-11-14T02:45:27,015 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ef754dfb5264c1782757434b94bbbd0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731552323723 2024-11-14T02:45:27,021 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae5c683ab910cad12ebb3cb2593f891b#C#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-14T02:45:27,022 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/600e5e3ff3a145128c3c951c94857667 is 50, key is test_row_0/C:col10/1731552324919/Put/seqid=0 2024-11-14T02:45:27,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742191_1367 (size=12561) 2024-11-14T02:45:27,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:27,053 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114630ff28d34784e79b943da30b2624dfc_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114630ff28d34784e79b943da30b2624dfc_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:27,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/17dd3e58def04f99b83cc5467320af9f, store: [table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:27,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/17dd3e58def04f99b83cc5467320af9f is 175, key is test_row_0/A:col10/1731552325024/Put/seqid=0 2024-11-14T02:45:27,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742192_1368 (size=31255) 2024-11-14T02:45:27,058 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/17dd3e58def04f99b83cc5467320af9f 2024-11-14T02:45:27,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/4207e8e813084d85b4410066c582e863 is 50, key is test_row_0/B:col10/1731552325024/Put/seqid=0 2024-11-14T02:45:27,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742193_1369 (size=12301) 2024-11-14T02:45:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T02:45:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:27,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:27,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552387192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:27,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552387192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:27,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:27,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:27,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552387225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:27,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552387225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:27,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552387226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:27,233 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_2' in ae5c683ab910cad12ebb3cb2593f891b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:45:27,234 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_2' in ae5c683ab910cad12ebb3cb2593f891b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:45:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:27,234 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_0' in ae5c683ab910cad12ebb3cb2593f891b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:45:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:27,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552387305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:27,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552387305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:27,428 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/600e5e3ff3a145128c3c951c94857667 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/600e5e3ff3a145128c3c951c94857667 2024-11-14T02:45:27,433 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae5c683ab910cad12ebb3cb2593f891b/C of ae5c683ab910cad12ebb3cb2593f891b into 600e5e3ff3a145128c3c951c94857667(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:27,433 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:27,433 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., storeName=ae5c683ab910cad12ebb3cb2593f891b/C, priority=13, startTime=1731552326578; duration=0sec 2024-11-14T02:45:27,433 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:27,433 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae5c683ab910cad12ebb3cb2593f891b:C 2024-11-14T02:45:27,467 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/4207e8e813084d85b4410066c582e863 2024-11-14T02:45:27,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/c3e30f34e9394f4faba6c7a01f785138 is 50, key is test_row_0/C:col10/1731552325024/Put/seqid=0 2024-11-14T02:45:27,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742194_1370 (size=12301) 2024-11-14T02:45:27,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552387525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:27,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552387525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:27,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552387845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:27,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552387845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:27,880 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/c3e30f34e9394f4faba6c7a01f785138 2024-11-14T02:45:27,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/17dd3e58def04f99b83cc5467320af9f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/17dd3e58def04f99b83cc5467320af9f 2024-11-14T02:45:27,888 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/17dd3e58def04f99b83cc5467320af9f, entries=150, sequenceid=279, filesize=30.5 K 2024-11-14T02:45:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/4207e8e813084d85b4410066c582e863 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/4207e8e813084d85b4410066c582e863 2024-11-14T02:45:27,892 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/4207e8e813084d85b4410066c582e863, entries=150, sequenceid=279, filesize=12.0 K 2024-11-14T02:45:27,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/c3e30f34e9394f4faba6c7a01f785138 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c3e30f34e9394f4faba6c7a01f785138 2024-11-14T02:45:27,895 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c3e30f34e9394f4faba6c7a01f785138, entries=150, sequenceid=279, filesize=12.0 K 2024-11-14T02:45:27,896 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for ae5c683ab910cad12ebb3cb2593f891b in 1259ms, sequenceid=279, compaction requested=false 2024-11-14T02:45:27,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:27,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:27,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-14T02:45:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-14T02:45:27,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-14T02:45:27,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8700 sec 2024-11-14T02:45:27,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8740 sec 2024-11-14T02:45:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T02:45:28,175 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:28,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-14T02:45:28,177 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T02:45:28,178 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:28,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T02:45:28,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T02:45:28,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:28,330 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing ae5c683ab910cad12ebb3cb2593f891b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T02:45:28,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=A 2024-11-14T02:45:28,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:28,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=B 2024-11-14T02:45:28,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:28,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=C 2024-11-14T02:45:28,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:28,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111498fb9bb1c5a64c7a8b3b7f4217ae5930_ae5c683ab910cad12ebb3cb2593f891b is 50, key is test_row_0/A:col10/1731552327183/Put/seqid=0 2024-11-14T02:45:28,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742195_1371 (size=12454) 2024-11-14T02:45:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:28,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. as already flushing 2024-11-14T02:45:28,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731552328392, max timestamp=1731552328426Num uniques 859; 2024-11-14T02:45:28,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:45:28,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731552328392, max timestamp=1731552328426Num uniques 859; 2024-11-14T02:45:28,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:45:28,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731552328392, max timestamp=1731552328426Num uniques 859; 2024-11-14T02:45:28,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:45:28,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552388483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T02:45:28,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552388487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:28,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552388595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:28,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552388605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:28,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:28,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:28,744 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111498fb9bb1c5a64c7a8b3b7f4217ae5930_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111498fb9bb1c5a64c7a8b3b7f4217ae5930_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:28,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/1bbe79b281334d6791ccd7f908e8c60f, store: [table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/1bbe79b281334d6791ccd7f908e8c60f is 175, key is test_row_0/A:col10/1731552327183/Put/seqid=0 2024-11-14T02:45:28,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742196_1372 (size=31255) 2024-11-14T02:45:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T02:45:28,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552388815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:28,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:28,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552388825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:29,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552389135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:29,148 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/1bbe79b281334d6791ccd7f908e8c60f 2024-11-14T02:45:29,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/f9716b01f4e942ebb794ba2a13650824 is 50, key is test_row_0/B:col10/1731552327183/Put/seqid=0 2024-11-14T02:45:29,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552389145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:29,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:29,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742197_1373 (size=12301) 2024-11-14T02:45:29,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52076 deadline: 1731552389245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:29,251 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_2' in ae5c683ab910cad12ebb3cb2593f891b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:45:29,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:29,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731552389255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:29,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52078 deadline: 1731552389255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:29,264 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_2' in ae5c683ab910cad12ebb3cb2593f891b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:45:29,264 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_0' in ae5c683ab910cad12ebb3cb2593f891b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:45:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T02:45:29,524 DEBUG [regionserver/e5dc8b5cf5e9:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-11-14T02:45:29,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/c85aa69e38b14b98b785291cfccb1ff8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/d344b6bf6555443b8c4a2c56b7502a86, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/c9d92d4eba684d08b0a25999b206df47, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/16636be1cf7f41cc9d2a940a83095a0f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/d4c80f08e023471b9550356784cc4ccb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/5fca256a29e2461b8f20755dbddd2820, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/9f341e6af414436f9889d1ef1046f7d8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/631fc88c934448959a0caf520ec23c2c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/58d3cbf0d05b4ea28013473647ec2b5b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/a3b03e616c8a48deb3d3cf196e106b4c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/f84d53182820426e8dbedbdd9ba14c9c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/edf63d6f7e2341af992ab2858baa6df5] to archive 2024-11-14T02:45:29,526 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:45:29,527 WARN [regionserver/e5dc8b5cf5e9:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-14T02:45:29,527 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/c85aa69e38b14b98b785291cfccb1ff8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/c85aa69e38b14b98b785291cfccb1ff8 2024-11-14T02:45:29,528 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/d344b6bf6555443b8c4a2c56b7502a86 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/d344b6bf6555443b8c4a2c56b7502a86 2024-11-14T02:45:29,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/c9d92d4eba684d08b0a25999b206df47 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/c9d92d4eba684d08b0a25999b206df47 2024-11-14T02:45:29,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/16636be1cf7f41cc9d2a940a83095a0f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/16636be1cf7f41cc9d2a940a83095a0f 2024-11-14T02:45:29,530 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/d4c80f08e023471b9550356784cc4ccb to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/d4c80f08e023471b9550356784cc4ccb 2024-11-14T02:45:29,531 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/5fca256a29e2461b8f20755dbddd2820 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/5fca256a29e2461b8f20755dbddd2820 2024-11-14T02:45:29,532 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/9f341e6af414436f9889d1ef1046f7d8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/9f341e6af414436f9889d1ef1046f7d8 2024-11-14T02:45:29,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/631fc88c934448959a0caf520ec23c2c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/631fc88c934448959a0caf520ec23c2c 2024-11-14T02:45:29,534 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/58d3cbf0d05b4ea28013473647ec2b5b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/58d3cbf0d05b4ea28013473647ec2b5b 2024-11-14T02:45:29,535 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/a3b03e616c8a48deb3d3cf196e106b4c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/a3b03e616c8a48deb3d3cf196e106b4c 2024-11-14T02:45:29,536 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/f84d53182820426e8dbedbdd9ba14c9c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/f84d53182820426e8dbedbdd9ba14c9c 2024-11-14T02:45:29,537 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/edf63d6f7e2341af992ab2858baa6df5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/edf63d6f7e2341af992ab2858baa6df5 2024-11-14T02:45:29,540 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/476235a588d04f629f5a108ba6a04b84, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d836554cfa7c42399d0127f159b57f0b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/4b30020161864461ba7630df6ee04f22, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/49a6a564cfe44bf1a37c897f77cdb027, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/0093c8a9b43c404ca14c0a3791c23d90, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/5560015b41224efb8224033d50370051, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/975cd801d260412ea962659f95bb9ad0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/37a2faba81fb4c09a26fa3b894831670, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d7efd121beb640b5a9de9663740069c4, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/30d21d99d4d6453792ce7d06c88d5f71, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d456de19003043c895ba78ee733d4dce, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d7297a4e43c9481388998257cb7c82e5] to archive 2024-11-14T02:45:29,540 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:45:29,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/476235a588d04f629f5a108ba6a04b84 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/476235a588d04f629f5a108ba6a04b84 2024-11-14T02:45:29,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d836554cfa7c42399d0127f159b57f0b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d836554cfa7c42399d0127f159b57f0b 2024-11-14T02:45:29,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/4b30020161864461ba7630df6ee04f22 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/4b30020161864461ba7630df6ee04f22 2024-11-14T02:45:29,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/49a6a564cfe44bf1a37c897f77cdb027 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/49a6a564cfe44bf1a37c897f77cdb027 2024-11-14T02:45:29,548 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/0093c8a9b43c404ca14c0a3791c23d90 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/0093c8a9b43c404ca14c0a3791c23d90 2024-11-14T02:45:29,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/5560015b41224efb8224033d50370051 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/5560015b41224efb8224033d50370051 2024-11-14T02:45:29,550 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/975cd801d260412ea962659f95bb9ad0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/975cd801d260412ea962659f95bb9ad0 2024-11-14T02:45:29,550 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/37a2faba81fb4c09a26fa3b894831670 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/37a2faba81fb4c09a26fa3b894831670 2024-11-14T02:45:29,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d7efd121beb640b5a9de9663740069c4 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d7efd121beb640b5a9de9663740069c4 2024-11-14T02:45:29,552 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/30d21d99d4d6453792ce7d06c88d5f71 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/30d21d99d4d6453792ce7d06c88d5f71 2024-11-14T02:45:29,553 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d456de19003043c895ba78ee733d4dce to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d456de19003043c895ba78ee733d4dce 2024-11-14T02:45:29,554 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d7297a4e43c9481388998257cb7c82e5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/d7297a4e43c9481388998257cb7c82e5 2024-11-14T02:45:29,556 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c6a85f0d24a149f9a9816e494284c314, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/12fa951b15aa41f598f830616d1efb9f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/22dd0060bbfa415fb9ce644692b6637f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/2dd7b467d4de461bb9fff9daba2b65b6, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/320061ac74f349afa50432f0d666fdc8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c8b0b7b77f594ce1913e35da704004e1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/57b2b809b66d4a748166c0575a0c32c7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/dbd175d10f964151a5b5301a52601554, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/1fc158116dfe46d79ee4f288c6ad89bb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/deaa7eb2c30748879514b47554cd3020, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/d15950aa62e247afa9e1edbdec6eed89, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/8ef754dfb5264c1782757434b94bbbd0] to archive 2024-11-14T02:45:29,557 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:45:29,558 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c6a85f0d24a149f9a9816e494284c314 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c6a85f0d24a149f9a9816e494284c314 2024-11-14T02:45:29,559 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/12fa951b15aa41f598f830616d1efb9f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/12fa951b15aa41f598f830616d1efb9f 2024-11-14T02:45:29,559 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/22dd0060bbfa415fb9ce644692b6637f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/22dd0060bbfa415fb9ce644692b6637f 2024-11-14T02:45:29,560 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/f9716b01f4e942ebb794ba2a13650824 2024-11-14T02:45:29,560 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/2dd7b467d4de461bb9fff9daba2b65b6 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/2dd7b467d4de461bb9fff9daba2b65b6 2024-11-14T02:45:29,561 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/320061ac74f349afa50432f0d666fdc8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/320061ac74f349afa50432f0d666fdc8 2024-11-14T02:45:29,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c8b0b7b77f594ce1913e35da704004e1 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c8b0b7b77f594ce1913e35da704004e1 2024-11-14T02:45:29,563 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/57b2b809b66d4a748166c0575a0c32c7 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/57b2b809b66d4a748166c0575a0c32c7 2024-11-14T02:45:29,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/dbd175d10f964151a5b5301a52601554 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/dbd175d10f964151a5b5301a52601554 2024-11-14T02:45:29,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/d84fd39e48cd45a9826d62385460d902 is 50, key is test_row_0/C:col10/1731552327183/Put/seqid=0 2024-11-14T02:45:29,567 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/1fc158116dfe46d79ee4f288c6ad89bb to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/1fc158116dfe46d79ee4f288c6ad89bb 2024-11-14T02:45:29,572 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/deaa7eb2c30748879514b47554cd3020 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/deaa7eb2c30748879514b47554cd3020 2024-11-14T02:45:29,574 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/d15950aa62e247afa9e1edbdec6eed89 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/d15950aa62e247afa9e1edbdec6eed89 2024-11-14T02:45:29,575 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/8ef754dfb5264c1782757434b94bbbd0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/8ef754dfb5264c1782757434b94bbbd0 2024-11-14T02:45:29,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742198_1374 (size=12301) 2024-11-14T02:45:29,581 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/d84fd39e48cd45a9826d62385460d902 2024-11-14T02:45:29,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/1bbe79b281334d6791ccd7f908e8c60f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/1bbe79b281334d6791ccd7f908e8c60f 2024-11-14T02:45:29,587 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/1bbe79b281334d6791ccd7f908e8c60f, entries=150, sequenceid=293, filesize=30.5 K 2024-11-14T02:45:29,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/f9716b01f4e942ebb794ba2a13650824 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/f9716b01f4e942ebb794ba2a13650824 2024-11-14T02:45:29,593 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/f9716b01f4e942ebb794ba2a13650824, entries=150, sequenceid=293, filesize=12.0 K 2024-11-14T02:45:29,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/d84fd39e48cd45a9826d62385460d902 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/d84fd39e48cd45a9826d62385460d902 2024-11-14T02:45:29,596 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/d84fd39e48cd45a9826d62385460d902, entries=150, sequenceid=293, filesize=12.0 K 2024-11-14T02:45:29,597 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ae5c683ab910cad12ebb3cb2593f891b in 1268ms, sequenceid=293, compaction requested=true 2024-11-14T02:45:29,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:29,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:29,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-14T02:45:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-14T02:45:29,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-14T02:45:29,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4200 sec 2024-11-14T02:45:29,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.4240 sec 2024-11-14T02:45:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:29,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae5c683ab910cad12ebb3cb2593f891b 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-14T02:45:29,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=A 2024-11-14T02:45:29,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:29,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=B 2024-11-14T02:45:29,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:29,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=C 2024-11-14T02:45:29,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:29,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111447e9762ad0934774ba4a8ed844f12718_ae5c683ab910cad12ebb3cb2593f891b is 50, key is test_row_0/A:col10/1731552328486/Put/seqid=0 2024-11-14T02:45:29,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742199_1375 (size=14994) 2024-11-14T02:45:29,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552389690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:29,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552389695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:29,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552389805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:29,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552389815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:29,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:29,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:30,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552390025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:30,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552390035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:30,063 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:30,067 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111447e9762ad0934774ba4a8ed844f12718_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111447e9762ad0934774ba4a8ed844f12718_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:30,067 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/68509d1169fc458f80630dc9bfff5bad, store: [table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:30,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/68509d1169fc458f80630dc9bfff5bad is 175, key is test_row_0/A:col10/1731552328486/Put/seqid=0 2024-11-14T02:45:30,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742200_1376 (size=39949) 2024-11-14T02:45:30,260 INFO [Thread-1462 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1462. 2024-11-14T02:45:30,260 DEBUG [Thread-1462 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:30,260 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:30,260 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:30,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:30,263 INFO [Thread-1458 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1458. 2024-11-14T02:45:30,263 DEBUG [Thread-1458 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:30,263 DEBUG [Thread-1458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:30,263 DEBUG [Thread-1458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:30,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:30,263 INFO [Thread-1460 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1460. 2024-11-14T02:45:30,263 DEBUG [Thread-1460 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:30,263 DEBUG [Thread-1460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:30,263 DEBUG [Thread-1460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:30,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:30,273 INFO [Thread-1464 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1464. 2024-11-14T02:45:30,273 DEBUG [Thread-1464 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:30,273 DEBUG [Thread-1464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:30,273 DEBUG [Thread-1464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:30,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:30,275 INFO [Thread-1466 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1466. 2024-11-14T02:45:30,275 DEBUG [Thread-1466 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:30,275 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:30,275 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:30,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T02:45:30,326 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:30,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:30,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552390345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552390345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:30,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:30,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:30,472 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/68509d1169fc458f80630dc9bfff5bad 2024-11-14T02:45:30,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/fbbd01c06e344db491182eae27a34185 is 50, key is test_row_0/B:col10/1731552328486/Put/seqid=0 2024-11-14T02:45:30,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742201_1377 (size=12301) 2024-11-14T02:45:30,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:30,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52130 deadline: 1731552390855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52086 deadline: 1731552390856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:45:30,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/fbbd01c06e344db491182eae27a34185 2024-11-14T02:45:30,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/610282b13a87456b90b3071177030ff4 is 50, key is test_row_0/C:col10/1731552328486/Put/seqid=0 2024-11-14T02:45:30,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742202_1378 (size=12301) 2024-11-14T02:45:31,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/610282b13a87456b90b3071177030ff4 2024-11-14T02:45:31,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/68509d1169fc458f80630dc9bfff5bad as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/68509d1169fc458f80630dc9bfff5bad 2024-11-14T02:45:31,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/68509d1169fc458f80630dc9bfff5bad, entries=200, sequenceid=328, filesize=39.0 K 2024-11-14T02:45:31,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/fbbd01c06e344db491182eae27a34185 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/fbbd01c06e344db491182eae27a34185 2024-11-14T02:45:31,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/fbbd01c06e344db491182eae27a34185, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T02:45:31,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/610282b13a87456b90b3071177030ff4 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/610282b13a87456b90b3071177030ff4 2024-11-14T02:45:31,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/610282b13a87456b90b3071177030ff4, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T02:45:31,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for ae5c683ab910cad12ebb3cb2593f891b in 1677ms, sequenceid=328, compaction requested=true 2024-11-14T02:45:31,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:31,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae5c683ab910cad12ebb3cb2593f891b:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:31,329 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:45:31,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:31,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae5c683ab910cad12ebb3cb2593f891b:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:31,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:31,329 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:45:31,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae5c683ab910cad12ebb3cb2593f891b:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:31,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:31,330 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133974 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:45:31,330 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:45:31,330 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ae5c683ab910cad12ebb3cb2593f891b/B is initiating minor compaction (all files) 2024-11-14T02:45:31,330 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ae5c683ab910cad12ebb3cb2593f891b/A is initiating minor compaction (all files) 2024-11-14T02:45:31,330 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae5c683ab910cad12ebb3cb2593f891b/A in TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:31,330 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae5c683ab910cad12ebb3cb2593f891b/B in TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:31,330 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/6c870ddaa7e34d2e91655c40df467566, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/4207e8e813084d85b4410066c582e863, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/f9716b01f4e942ebb794ba2a13650824, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/fbbd01c06e344db491182eae27a34185] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp, totalSize=48.3 K 2024-11-14T02:45:31,330 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/755ed8c8aae84b5fa33fc10dca4e097e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/17dd3e58def04f99b83cc5467320af9f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/1bbe79b281334d6791ccd7f908e8c60f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/68509d1169fc458f80630dc9bfff5bad] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp, totalSize=130.8 K 2024-11-14T02:45:31,330 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:31,331 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/755ed8c8aae84b5fa33fc10dca4e097e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/17dd3e58def04f99b83cc5467320af9f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/1bbe79b281334d6791ccd7f908e8c60f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/68509d1169fc458f80630dc9bfff5bad] 2024-11-14T02:45:31,331 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c870ddaa7e34d2e91655c40df467566, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731552323723 2024-11-14T02:45:31,331 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 755ed8c8aae84b5fa33fc10dca4e097e, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731552323723 2024-11-14T02:45:31,331 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 4207e8e813084d85b4410066c582e863, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731552325024 2024-11-14T02:45:31,331 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17dd3e58def04f99b83cc5467320af9f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731552325024 2024-11-14T02:45:31,331 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting f9716b01f4e942ebb794ba2a13650824, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731552327183 2024-11-14T02:45:31,331 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bbe79b281334d6791ccd7f908e8c60f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731552327183 2024-11-14T02:45:31,331 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting fbbd01c06e344db491182eae27a34185, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731552328481 2024-11-14T02:45:31,331 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68509d1169fc458f80630dc9bfff5bad, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731552328475 2024-11-14T02:45:31,339 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:31,339 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae5c683ab910cad12ebb3cb2593f891b#B#compaction#317 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:31,339 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/a74a35462c024330bc6017e744e77554 is 50, key is test_row_0/B:col10/1731552328486/Put/seqid=0 2024-11-14T02:45:31,340 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411148c14926f63cf46ee8f28182d1cfbdc43_ae5c683ab910cad12ebb3cb2593f891b store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:31,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742203_1379 (size=12439) 2024-11-14T02:45:31,351 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411148c14926f63cf46ee8f28182d1cfbdc43_ae5c683ab910cad12ebb3cb2593f891b, store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:31,351 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148c14926f63cf46ee8f28182d1cfbdc43_ae5c683ab910cad12ebb3cb2593f891b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:31,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742204_1380 (size=4469) 2024-11-14T02:45:31,752 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/a74a35462c024330bc6017e744e77554 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/a74a35462c024330bc6017e744e77554 2024-11-14T02:45:31,755 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae5c683ab910cad12ebb3cb2593f891b#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:31,756 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/385a5258305f4a59a34e4d6c7c58f900 is 175, key is test_row_0/A:col10/1731552328486/Put/seqid=0 2024-11-14T02:45:31,757 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae5c683ab910cad12ebb3cb2593f891b/B of ae5c683ab910cad12ebb3cb2593f891b into a74a35462c024330bc6017e744e77554(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:31,757 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:31,757 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., storeName=ae5c683ab910cad12ebb3cb2593f891b/B, priority=12, startTime=1731552331329; duration=0sec 2024-11-14T02:45:31,757 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:31,757 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae5c683ab910cad12ebb3cb2593f891b:B 2024-11-14T02:45:31,757 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:45:31,758 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:45:31,758 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ae5c683ab910cad12ebb3cb2593f891b/C is initiating minor compaction (all files) 2024-11-14T02:45:31,758 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae5c683ab910cad12ebb3cb2593f891b/C in TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:31,759 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/600e5e3ff3a145128c3c951c94857667, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c3e30f34e9394f4faba6c7a01f785138, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/d84fd39e48cd45a9826d62385460d902, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/610282b13a87456b90b3071177030ff4] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp, totalSize=48.3 K 2024-11-14T02:45:31,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742205_1381 (size=31393) 2024-11-14T02:45:31,759 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 600e5e3ff3a145128c3c951c94857667, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731552323723 2024-11-14T02:45:31,759 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e30f34e9394f4faba6c7a01f785138, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731552325024 2024-11-14T02:45:31,759 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting d84fd39e48cd45a9826d62385460d902, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731552327183 2024-11-14T02:45:31,759 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 610282b13a87456b90b3071177030ff4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731552328481 2024-11-14T02:45:31,768 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae5c683ab910cad12ebb3cb2593f891b#C#compaction#319 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:31,768 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/2d26521ba40d4a679c7ceeb58165c01c is 50, key is test_row_0/C:col10/1731552328486/Put/seqid=0 2024-11-14T02:45:31,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742206_1382 (size=12439) 2024-11-14T02:45:31,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:31,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae5c683ab910cad12ebb3cb2593f891b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T02:45:31,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=A 2024-11-14T02:45:31,869 INFO [Thread-1449 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1449. 2024-11-14T02:45:31,869 INFO [Thread-1451 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1451. 2024-11-14T02:45:31,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:31,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=B 2024-11-14T02:45:31,869 DEBUG [Thread-1449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:31,869 DEBUG [Thread-1451 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:31,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:31,869 DEBUG [Thread-1449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:31,869 DEBUG [Thread-1451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:31,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=C 2024-11-14T02:45:31,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:31,870 DEBUG [Thread-1449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:31,870 DEBUG [Thread-1451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:31,870 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:31,870 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:31,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141c50e53548e74095badce0252973165a_ae5c683ab910cad12ebb3cb2593f891b is 50, key is test_row_0/A:col10/1731552329671/Put/seqid=0 2024-11-14T02:45:31,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742207_1383 (size=12454) 2024-11-14T02:45:32,169 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/385a5258305f4a59a34e4d6c7c58f900 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/385a5258305f4a59a34e4d6c7c58f900 2024-11-14T02:45:32,176 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae5c683ab910cad12ebb3cb2593f891b/A of ae5c683ab910cad12ebb3cb2593f891b into 385a5258305f4a59a34e4d6c7c58f900(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:32,176 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:32,176 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., storeName=ae5c683ab910cad12ebb3cb2593f891b/A, priority=12, startTime=1731552331329; duration=0sec 2024-11-14T02:45:32,176 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:32,176 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae5c683ab910cad12ebb3cb2593f891b:A 2024-11-14T02:45:32,177 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/2d26521ba40d4a679c7ceeb58165c01c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/2d26521ba40d4a679c7ceeb58165c01c 2024-11-14T02:45:32,183 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae5c683ab910cad12ebb3cb2593f891b/C of ae5c683ab910cad12ebb3cb2593f891b into 2d26521ba40d4a679c7ceeb58165c01c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:32,183 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:32,183 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., storeName=ae5c683ab910cad12ebb3cb2593f891b/C, priority=12, startTime=1731552331329; duration=0sec 2024-11-14T02:45:32,183 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:32,183 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae5c683ab910cad12ebb3cb2593f891b:C 2024-11-14T02:45:32,284 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:32,293 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141c50e53548e74095badce0252973165a_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141c50e53548e74095badce0252973165a_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:32,294 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/5bdfc32557a34ba7b32b34c73d980470, store: [table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:32,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/5bdfc32557a34ba7b32b34c73d980470 is 175, key is test_row_0/A:col10/1731552329671/Put/seqid=0 2024-11-14T02:45:32,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742208_1384 (size=31255) 2024-11-14T02:45:32,700 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=341, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/5bdfc32557a34ba7b32b34c73d980470 2024-11-14T02:45:32,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/da3bb1cd2d7b4924834414e456e739f9 is 50, key is test_row_0/B:col10/1731552329671/Put/seqid=0 2024-11-14T02:45:32,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742209_1385 (size=12301) 2024-11-14T02:45:33,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/da3bb1cd2d7b4924834414e456e739f9 2024-11-14T02:45:33,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/5efd8a313de744b9b46347b0b0b651bd is 50, key is test_row_0/C:col10/1731552329671/Put/seqid=0 2024-11-14T02:45:33,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742210_1386 (size=12301) 2024-11-14T02:45:33,289 INFO [Thread-1455 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1455. 2024-11-14T02:45:33,289 DEBUG [Thread-1455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:33,289 DEBUG [Thread-1455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:33,289 DEBUG [Thread-1455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:33,290 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:33,297 INFO [Thread-1453 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1453. 2024-11-14T02:45:33,298 DEBUG [Thread-1453 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:33,298 DEBUG [Thread-1453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:33,298 DEBUG [Thread-1453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:33,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:33,298 INFO [Thread-1447 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1447. 2024-11-14T02:45:33,298 DEBUG [Thread-1447 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:33,298 DEBUG [Thread-1447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:33,298 DEBUG [Thread-1447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:33,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1052 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3152 rows 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1057 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3171 rows 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1056 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3168 rows 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1054 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3160 rows 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1054 2024-11-14T02:45:33,298 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3159 rows 2024-11-14T02:45:33,299 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:45:33,299 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:45:33,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:33,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:33,299 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:33,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T02:45:33,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T02:45:33,302 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552333302"}]},"ts":"1731552333302"} 2024-11-14T02:45:33,304 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T02:45:33,304 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T02:45:33,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:45:33,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b, UNASSIGN}] 2024-11-14T02:45:33,306 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b, UNASSIGN 2024-11-14T02:45:33,307 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=ae5c683ab910cad12ebb3cb2593f891b, regionState=CLOSING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:33,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b, UNASSIGN because future has completed 2024-11-14T02:45:33,309 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:45:33,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:45:33,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T02:45:33,462 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(122): Close ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:33,463 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:45:33,463 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1722): Closing ae5c683ab910cad12ebb3cb2593f891b, disabling compactions & flushes 2024-11-14T02:45:33,463 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:33,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/5efd8a313de744b9b46347b0b0b651bd 2024-11-14T02:45:33,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/5bdfc32557a34ba7b32b34c73d980470 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/5bdfc32557a34ba7b32b34c73d980470 2024-11-14T02:45:33,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/5bdfc32557a34ba7b32b34c73d980470, entries=150, sequenceid=341, filesize=30.5 K 2024-11-14T02:45:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/da3bb1cd2d7b4924834414e456e739f9 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/da3bb1cd2d7b4924834414e456e739f9 2024-11-14T02:45:33,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/da3bb1cd2d7b4924834414e456e739f9, entries=150, sequenceid=341, filesize=12.0 K 2024-11-14T02:45:33,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/5efd8a313de744b9b46347b0b0b651bd as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/5efd8a313de744b9b46347b0b0b651bd 2024-11-14T02:45:33,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/5efd8a313de744b9b46347b0b0b651bd, entries=150, sequenceid=341, filesize=12.0 K 2024-11-14T02:45:33,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for ae5c683ab910cad12ebb3cb2593f891b in 1696ms, sequenceid=341, compaction requested=false 2024-11-14T02:45:33,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae5c683ab910cad12ebb3cb2593f891b: 2024-11-14T02:45:33,564 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:33,564 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:33,564 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. after waiting 0 ms 2024-11-14T02:45:33,564 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:33,565 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(2902): Flushing ae5c683ab910cad12ebb3cb2593f891b 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-14T02:45:33,565 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=A 2024-11-14T02:45:33,565 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:33,565 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=B 2024-11-14T02:45:33,565 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:33,565 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae5c683ab910cad12ebb3cb2593f891b, store=C 2024-11-14T02:45:33,565 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:33,570 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c8f5437cbd95481e9b1fe2786e950b70_ae5c683ab910cad12ebb3cb2593f891b is 50, key is test_row_0/A:col10/1731552333296/Put/seqid=0 2024-11-14T02:45:33,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742211_1387 (size=9914) 2024-11-14T02:45:33,574 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:33,577 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c8f5437cbd95481e9b1fe2786e950b70_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c8f5437cbd95481e9b1fe2786e950b70_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:33,578 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/b03e499fd336485b92ddca126cc4fd50, store: [table=TestAcidGuarantees family=A region=ae5c683ab910cad12ebb3cb2593f891b] 2024-11-14T02:45:33,578 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/b03e499fd336485b92ddca126cc4fd50 is 175, key is test_row_0/A:col10/1731552333296/Put/seqid=0 2024-11-14T02:45:33,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742212_1388 (size=22561) 2024-11-14T02:45:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T02:45:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T02:45:33,983 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/b03e499fd336485b92ddca126cc4fd50 2024-11-14T02:45:33,995 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/5b2fb51ce66744d2ba0ad788f1f08f72 is 50, key is test_row_0/B:col10/1731552333296/Put/seqid=0 2024-11-14T02:45:33,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742213_1389 (size=9857) 2024-11-14T02:45:34,401 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/5b2fb51ce66744d2ba0ad788f1f08f72 2024-11-14T02:45:34,413 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/52df7202a47a4ccdb778c59f192cf88c is 50, key is test_row_0/C:col10/1731552333296/Put/seqid=0 2024-11-14T02:45:34,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742214_1390 (size=9857) 2024-11-14T02:45:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T02:45:34,817 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/52df7202a47a4ccdb778c59f192cf88c 2024-11-14T02:45:34,826 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/A/b03e499fd336485b92ddca126cc4fd50 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/b03e499fd336485b92ddca126cc4fd50 2024-11-14T02:45:34,832 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/b03e499fd336485b92ddca126cc4fd50, entries=100, sequenceid=349, filesize=22.0 K 2024-11-14T02:45:34,833 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/B/5b2fb51ce66744d2ba0ad788f1f08f72 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/5b2fb51ce66744d2ba0ad788f1f08f72 2024-11-14T02:45:34,838 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/5b2fb51ce66744d2ba0ad788f1f08f72, entries=100, sequenceid=349, filesize=9.6 K 2024-11-14T02:45:34,838 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/.tmp/C/52df7202a47a4ccdb778c59f192cf88c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/52df7202a47a4ccdb778c59f192cf88c 2024-11-14T02:45:34,841 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/52df7202a47a4ccdb778c59f192cf88c, entries=100, sequenceid=349, filesize=9.6 K 2024-11-14T02:45:34,842 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for ae5c683ab910cad12ebb3cb2593f891b in 1278ms, sequenceid=349, compaction requested=true 2024-11-14T02:45:34,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/755ed8c8aae84b5fa33fc10dca4e097e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/17dd3e58def04f99b83cc5467320af9f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/1bbe79b281334d6791ccd7f908e8c60f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/68509d1169fc458f80630dc9bfff5bad] to archive 2024-11-14T02:45:34,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:45:34,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/755ed8c8aae84b5fa33fc10dca4e097e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/755ed8c8aae84b5fa33fc10dca4e097e 2024-11-14T02:45:34,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/17dd3e58def04f99b83cc5467320af9f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/17dd3e58def04f99b83cc5467320af9f 2024-11-14T02:45:34,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/1bbe79b281334d6791ccd7f908e8c60f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/1bbe79b281334d6791ccd7f908e8c60f 2024-11-14T02:45:34,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/68509d1169fc458f80630dc9bfff5bad to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/68509d1169fc458f80630dc9bfff5bad 2024-11-14T02:45:34,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/6c870ddaa7e34d2e91655c40df467566, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/4207e8e813084d85b4410066c582e863, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/f9716b01f4e942ebb794ba2a13650824, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/fbbd01c06e344db491182eae27a34185] to archive 2024-11-14T02:45:34,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:45:34,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/6c870ddaa7e34d2e91655c40df467566 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/6c870ddaa7e34d2e91655c40df467566 2024-11-14T02:45:34,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/4207e8e813084d85b4410066c582e863 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/4207e8e813084d85b4410066c582e863 2024-11-14T02:45:34,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/f9716b01f4e942ebb794ba2a13650824 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/f9716b01f4e942ebb794ba2a13650824 2024-11-14T02:45:34,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/fbbd01c06e344db491182eae27a34185 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/fbbd01c06e344db491182eae27a34185 2024-11-14T02:45:34,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/600e5e3ff3a145128c3c951c94857667, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c3e30f34e9394f4faba6c7a01f785138, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/d84fd39e48cd45a9826d62385460d902, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/610282b13a87456b90b3071177030ff4] to archive 2024-11-14T02:45:34,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:45:34,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/600e5e3ff3a145128c3c951c94857667 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/600e5e3ff3a145128c3c951c94857667 2024-11-14T02:45:34,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c3e30f34e9394f4faba6c7a01f785138 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/c3e30f34e9394f4faba6c7a01f785138 2024-11-14T02:45:34,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/d84fd39e48cd45a9826d62385460d902 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/d84fd39e48cd45a9826d62385460d902 2024-11-14T02:45:34,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/610282b13a87456b90b3071177030ff4 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/610282b13a87456b90b3071177030ff4 2024-11-14T02:45:34,859 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/recovered.edits/352.seqid, newMaxSeqId=352, maxSeqId=4 2024-11-14T02:45:34,859 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. 2024-11-14T02:45:34,859 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1676): Region close journal for ae5c683ab910cad12ebb3cb2593f891b: Waiting for close lock at 1731552333463Running coprocessor pre-close hooks at 1731552333463Disabling compacts and flushes for region at 1731552333463Disabling writes for close at 1731552333564 (+101 ms)Obtaining lock to block concurrent updates at 1731552333565 (+1 ms)Preparing flush snapshotting stores in ae5c683ab910cad12ebb3cb2593f891b at 1731552333565Finished memstore snapshotting TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731552333565Flushing stores of TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b. at 1731552333565Flushing ae5c683ab910cad12ebb3cb2593f891b/A: creating writer at 1731552333565Flushing mob file ae5c683ab910cad12ebb3cb2593f891b/A: appending metadata at 1731552333570 (+5 ms)Flushing mob file ae5c683ab910cad12ebb3cb2593f891b/A: closing flushed file at 1731552333570Flushing ae5c683ab910cad12ebb3cb2593f891b/A: appending metadata at 1731552333578 (+8 ms)Flushing ae5c683ab910cad12ebb3cb2593f891b/A: closing flushed file at 1731552333578Flushing ae5c683ab910cad12ebb3cb2593f891b/B: creating writer at 1731552333991 (+413 ms)Flushing ae5c683ab910cad12ebb3cb2593f891b/B: appending metadata at 1731552333994 (+3 ms)Flushing ae5c683ab910cad12ebb3cb2593f891b/B: closing flushed file at 1731552333994Flushing ae5c683ab910cad12ebb3cb2593f891b/C: creating writer at 1731552334408 (+414 ms)Flushing ae5c683ab910cad12ebb3cb2593f891b/C: appending metadata at 1731552334412 (+4 ms)Flushing ae5c683ab910cad12ebb3cb2593f891b/C: closing flushed file at 1731552334412Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c4925e4: reopening flushed file at 1731552334825 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e78656a: reopening flushed file at 1731552334832 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@465115bb: reopening flushed file at 1731552334838 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for ae5c683ab910cad12ebb3cb2593f891b in 1278ms, sequenceid=349, compaction requested=true at 1731552334842 (+4 ms)Writing region close event to WAL at 1731552334857 (+15 ms)Running coprocessor post-close hooks at 1731552334859 (+2 ms)Closed at 1731552334859 2024-11-14T02:45:34,861 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(157): Closed ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:34,861 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=ae5c683ab910cad12ebb3cb2593f891b, regionState=CLOSED 2024-11-14T02:45:34,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:45:34,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-14T02:45:34,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseRegionProcedure ae5c683ab910cad12ebb3cb2593f891b, server=e5dc8b5cf5e9,46387,1731552206828 in 1.5540 sec 2024-11-14T02:45:34,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-14T02:45:34,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae5c683ab910cad12ebb3cb2593f891b, UNASSIGN in 1.5580 sec 2024-11-14T02:45:34,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-14T02:45:34,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5620 sec 2024-11-14T02:45:34,868 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552334868"}]},"ts":"1731552334868"} 2024-11-14T02:45:34,869 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T02:45:34,869 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T02:45:34,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5700 sec 2024-11-14T02:45:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T02:45:35,447 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:35,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T02:45:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:35,509 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T02:45:35,511 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=105, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:35,516 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,521 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/recovered.edits] 2024-11-14T02:45:35,524 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/385a5258305f4a59a34e4d6c7c58f900 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/385a5258305f4a59a34e4d6c7c58f900 2024-11-14T02:45:35,526 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/5bdfc32557a34ba7b32b34c73d980470 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/5bdfc32557a34ba7b32b34c73d980470 2024-11-14T02:45:35,527 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/b03e499fd336485b92ddca126cc4fd50 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/A/b03e499fd336485b92ddca126cc4fd50 2024-11-14T02:45:35,530 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/5b2fb51ce66744d2ba0ad788f1f08f72 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/5b2fb51ce66744d2ba0ad788f1f08f72 2024-11-14T02:45:35,531 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/a74a35462c024330bc6017e744e77554 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/a74a35462c024330bc6017e744e77554 2024-11-14T02:45:35,533 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/da3bb1cd2d7b4924834414e456e739f9 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/B/da3bb1cd2d7b4924834414e456e739f9 2024-11-14T02:45:35,536 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/2d26521ba40d4a679c7ceeb58165c01c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/2d26521ba40d4a679c7ceeb58165c01c 2024-11-14T02:45:35,537 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/52df7202a47a4ccdb778c59f192cf88c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/52df7202a47a4ccdb778c59f192cf88c 2024-11-14T02:45:35,539 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/5efd8a313de744b9b46347b0b0b651bd to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/C/5efd8a313de744b9b46347b0b0b651bd 2024-11-14T02:45:35,542 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/recovered.edits/352.seqid to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b/recovered.edits/352.seqid 2024-11-14T02:45:35,543 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,543 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T02:45:35,544 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T02:45:35,545 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T02:45:35,550 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114027c2f6bed6e4f4dab2a9dbd727591d2_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114027c2f6bed6e4f4dab2a9dbd727591d2_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,552 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114071a6e353d2f4047936a91dd048b48e6_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114071a6e353d2f4047936a91dd048b48e6_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,554 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114161291e09b9f465ca3fa6ccadc4b1fa1_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114161291e09b9f465ca3fa6ccadc4b1fa1_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,556 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141c50e53548e74095badce0252973165a_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141c50e53548e74095badce0252973165a_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,557 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142e844b9e3f4141cb8caf54bbc4abc125_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142e844b9e3f4141cb8caf54bbc4abc125_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,559 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111447e9762ad0934774ba4a8ed844f12718_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111447e9762ad0934774ba4a8ed844f12718_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,561 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144aac2b45266f4e19a0698c5d1d46a83c_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144aac2b45266f4e19a0698c5d1d46a83c_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,562 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111453c6911bb6b54716acf3cbcf389056ac_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111453c6911bb6b54716acf3cbcf389056ac_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,563 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111461261a7f3ea7410d9eeebb24e4ea8580_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111461261a7f3ea7410d9eeebb24e4ea8580_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,565 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114630ff28d34784e79b943da30b2624dfc_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114630ff28d34784e79b943da30b2624dfc_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,567 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111498fb9bb1c5a64c7a8b3b7f4217ae5930_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111498fb9bb1c5a64c7a8b3b7f4217ae5930_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,568 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b00a2381aaec4198a6b2eef0bf7bcfeb_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b00a2381aaec4198a6b2eef0bf7bcfeb_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,569 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c8f5437cbd95481e9b1fe2786e950b70_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c8f5437cbd95481e9b1fe2786e950b70_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,571 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e12cca5843184c80992cd1efa53cffd2_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e12cca5843184c80992cd1efa53cffd2_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,572 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fd3e8710bb6745b5b995aec1e0cacb04_ae5c683ab910cad12ebb3cb2593f891b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fd3e8710bb6745b5b995aec1e0cacb04_ae5c683ab910cad12ebb3cb2593f891b 2024-11-14T02:45:35,573 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T02:45:35,575 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=105, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:35,578 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T02:45:35,580 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T02:45:35,581 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=105, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:35,581 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T02:45:35,582 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731552335581"}]},"ts":"9223372036854775807"} 2024-11-14T02:45:35,583 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T02:45:35,583 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ae5c683ab910cad12ebb3cb2593f891b, NAME => 'TestAcidGuarantees,,1731552307024.ae5c683ab910cad12ebb3cb2593f891b.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T02:45:35,584 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T02:45:35,584 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731552335584"}]},"ts":"9223372036854775807"} 2024-11-14T02:45:35,585 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T02:45:35,586 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=105, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:35,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 138 msec 2024-11-14T02:45:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T02:45:35,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T02:45:35,620 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:35,629 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 248) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-521717827_22 at /127.0.0.1:55390 [Waiting for operation #192] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-242810252_22 at /127.0.0.1:45154 [Waiting for operation #147] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=348 (was 336) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9805 (was 9827) 2024-11-14T02:45:35,640 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=348, ProcessCount=11, AvailableMemoryMB=9805 2024-11-14T02:45:35,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:45:35,641 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:45:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T02:45:35,643 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T02:45:35,643 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:35,643 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 106 2024-11-14T02:45:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-14T02:45:35,644 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T02:45:35,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742215_1391 (size=1061) 2024-11-14T02:45:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-14T02:45:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-14T02:45:36,112 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1 2024-11-14T02:45:36,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742216_1392 (size=53) 2024-11-14T02:45:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-14T02:45:36,489 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T02:45:36,523 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:45:36,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ac0d8410ac14c280aa06089ab1aa8285, disabling compactions & flushes 2024-11-14T02:45:36,524 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:36,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:36,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. after waiting 0 ms 2024-11-14T02:45:36,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:36,524 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:36,524 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ac0d8410ac14c280aa06089ab1aa8285: Waiting for close lock at 1731552336524Disabling compacts and flushes for region at 1731552336524Disabling writes for close at 1731552336524Writing region close event to WAL at 1731552336524Closed at 1731552336524 2024-11-14T02:45:36,527 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T02:45:36,528 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731552336527"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731552336527"}]},"ts":"1731552336527"} 2024-11-14T02:45:36,532 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T02:45:36,533 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T02:45:36,534 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552336533"}]},"ts":"1731552336533"} 2024-11-14T02:45:36,536 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T02:45:36,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac0d8410ac14c280aa06089ab1aa8285, ASSIGN}] 2024-11-14T02:45:36,538 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac0d8410ac14c280aa06089ab1aa8285, ASSIGN 2024-11-14T02:45:36,539 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac0d8410ac14c280aa06089ab1aa8285, ASSIGN; state=OFFLINE, location=e5dc8b5cf5e9,46387,1731552206828; forceNewPlan=false, retain=false 2024-11-14T02:45:36,690 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=ac0d8410ac14c280aa06089ab1aa8285, regionState=OPENING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:36,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac0d8410ac14c280aa06089ab1aa8285, ASSIGN because future has completed 2024-11-14T02:45:36,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:45:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-14T02:45:36,855 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:36,855 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:45:36,856 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:36,857 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:45:36,857 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:36,857 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:36,858 INFO [StoreOpener-ac0d8410ac14c280aa06089ab1aa8285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:36,860 INFO [StoreOpener-ac0d8410ac14c280aa06089ab1aa8285-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:45:36,860 INFO [StoreOpener-ac0d8410ac14c280aa06089ab1aa8285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac0d8410ac14c280aa06089ab1aa8285 columnFamilyName A 2024-11-14T02:45:36,860 DEBUG [StoreOpener-ac0d8410ac14c280aa06089ab1aa8285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:36,861 INFO [StoreOpener-ac0d8410ac14c280aa06089ab1aa8285-1 {}] regionserver.HStore(327): Store=ac0d8410ac14c280aa06089ab1aa8285/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:36,861 INFO [StoreOpener-ac0d8410ac14c280aa06089ab1aa8285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:36,862 INFO [StoreOpener-ac0d8410ac14c280aa06089ab1aa8285-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:45:36,862 INFO [StoreOpener-ac0d8410ac14c280aa06089ab1aa8285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac0d8410ac14c280aa06089ab1aa8285 columnFamilyName B 2024-11-14T02:45:36,862 DEBUG [StoreOpener-ac0d8410ac14c280aa06089ab1aa8285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:36,863 INFO [StoreOpener-ac0d8410ac14c280aa06089ab1aa8285-1 {}] regionserver.HStore(327): Store=ac0d8410ac14c280aa06089ab1aa8285/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:36,863 INFO [StoreOpener-ac0d8410ac14c280aa06089ab1aa8285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:36,864 INFO [StoreOpener-ac0d8410ac14c280aa06089ab1aa8285-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:45:36,864 INFO [StoreOpener-ac0d8410ac14c280aa06089ab1aa8285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac0d8410ac14c280aa06089ab1aa8285 columnFamilyName C 2024-11-14T02:45:36,864 DEBUG [StoreOpener-ac0d8410ac14c280aa06089ab1aa8285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:45:36,865 INFO [StoreOpener-ac0d8410ac14c280aa06089ab1aa8285-1 {}] regionserver.HStore(327): Store=ac0d8410ac14c280aa06089ab1aa8285/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:45:36,865 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:36,865 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:36,866 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:36,866 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:36,867 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:36,867 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:36,868 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:45:36,869 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:36,871 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:45:36,871 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened ac0d8410ac14c280aa06089ab1aa8285; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68328822, jitterRate=0.018178790807724}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:45:36,872 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:36,872 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for ac0d8410ac14c280aa06089ab1aa8285: Running coprocessor pre-open hook at 1731552336857Writing region info on filesystem at 1731552336857Initializing all the Stores at 1731552336858 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552336858Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552336858Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552336858Cleaning up temporary data from old regions at 1731552336867 (+9 ms)Running coprocessor post-open hooks at 1731552336872 (+5 ms)Region opened successfully at 1731552336872 2024-11-14T02:45:36,873 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., pid=108, masterSystemTime=1731552336848 2024-11-14T02:45:36,874 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:36,874 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:36,875 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=ac0d8410ac14c280aa06089ab1aa8285, regionState=OPEN, openSeqNum=2, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:36,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:45:36,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-14T02:45:36,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; OpenRegionProcedure ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 in 182 msec 2024-11-14T02:45:36,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-14T02:45:36,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac0d8410ac14c280aa06089ab1aa8285, ASSIGN in 343 msec 2024-11-14T02:45:36,881 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T02:45:36,882 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552336881"}]},"ts":"1731552336881"} 2024-11-14T02:45:36,883 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T02:45:36,884 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T02:45:36,886 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2430 sec 2024-11-14T02:45:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-14T02:45:37,796 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:37,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72240ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,798 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:37,798 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:37,800 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:37,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ebd71ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:37,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:37,803 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22f9fc7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,805 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:37,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:37,810 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:45:37,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:45:37,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@122e870c 2024-11-14T02:45:37,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:45:37,813 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:45:37,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:37,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:37,816 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:37,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55446532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:37,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:37,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:37,819 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:37,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@566487c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,821 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:37,821 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:37,822 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11027dbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:37,823 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5479de8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,825 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:37,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:37,826 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@96d93a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:37,827 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7543b243, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:37,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:37,829 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:37,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:37,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:37,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d3df97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:37,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:37,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:37,831 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8c70f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:37,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:37,834 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ae8b609, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:37,835 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:37,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@131404cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:37,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:37,837 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6225aeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:37,839 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1883511f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:37,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:37,841 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69da0f02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:37,842 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e3fca7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,843 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:37,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:37,844 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6991f27c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:37,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:37,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:37,845 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:37,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f1ccc6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,846 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:37,846 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:37,847 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:37,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:37,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:37,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eb3471e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:37,848 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@721affb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663bb5d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:45:37,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:45:37,850 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:45:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:45:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:45:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d58cb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:45:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:45:37,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:37,851 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:45:37,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ca30800, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:45:37,852 DEBUG [Thread-1802 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:37,852 DEBUG [Thread-1787 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:37,852 DEBUG [Thread-1798 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:37,852 DEBUG [Thread-1789 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:37,852 DEBUG [Thread-1791 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:37,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:37,853 DEBUG [Thread-1793 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:37,853 DEBUG [Thread-1804 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:37,854 DEBUG [Thread-1800 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:37,854 DEBUG [Thread-1795 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-14T02:45:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:37,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:37,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:37,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:37,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T02:45:37,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:37,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:37,856 DEBUG [Thread-1806 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:45:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:37,857 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:45:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:45:37,858 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:37,858 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:37,858 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:45:37,858 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:45:37,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:37,859 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:45:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:45:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:45:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:45:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:45:37,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:45:37,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:45:37,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:45:37,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2] 2024-11-14T02:45:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:37,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:45:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:37,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/37d967b862454008ac2b41cea52b9f91 is 50, key is test_row_0/A:col10/1731552337865/Put/seqid=0 2024-11-14T02:45:37,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742217_1393 (size=16681) 2024-11-14T02:45:37,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731552337887, max timestamp=1731552337906Num uniques 859; 2024-11-14T02:45:37,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731552337887, max timestamp=1731552337906Num uniques 859; 2024-11-14T02:45:37,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731552337887, max timestamp=1731552337906Num uniques 859; 2024-11-14T02:45:37,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T02:45:37,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T02:45:37,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T02:45:37,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552397932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:37,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552397933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:37,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552397934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:37,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552397939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:37,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552397946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:37,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T02:45:38,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T02:45:38,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:38,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:38,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:38,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552398055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:38,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552398055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552398055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552398055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:38,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552398055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T02:45:38,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:38,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:38,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:38,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T02:45:38,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552398265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:38,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552398265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552398265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552398275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:38,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552398275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/37d967b862454008ac2b41cea52b9f91 2024-11-14T02:45:38,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T02:45:38,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:38,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:38,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:38,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/e2f18a87f3634aa3833c09948d31d406 is 50, key is test_row_0/B:col10/1731552337865/Put/seqid=0 2024-11-14T02:45:38,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742218_1394 (size=12001) 2024-11-14T02:45:38,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T02:45:38,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:38,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:38,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:38,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T02:45:38,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552398585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:38,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552398585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552398585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552398585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552398595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:38,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T02:45:38,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:38,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:38,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:38,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/e2f18a87f3634aa3833c09948d31d406 2024-11-14T02:45:38,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/d4946b7a71e745239d06134c423653f5 is 50, key is test_row_0/C:col10/1731552337865/Put/seqid=0 2024-11-14T02:45:38,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742219_1395 (size=12001) 2024-11-14T02:45:38,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T02:45:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:38,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T02:45:38,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:38,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:38,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:38,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T02:45:39,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T02:45:39,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:39,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:39,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:39,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:39,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:39,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552399095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:39,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552399105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:39,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552399105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552399106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552399106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:39,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/d4946b7a71e745239d06134c423653f5 2024-11-14T02:45:39,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/37d967b862454008ac2b41cea52b9f91 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/37d967b862454008ac2b41cea52b9f91 2024-11-14T02:45:39,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/37d967b862454008ac2b41cea52b9f91, entries=250, sequenceid=12, filesize=16.3 K 2024-11-14T02:45:39,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/e2f18a87f3634aa3833c09948d31d406 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/e2f18a87f3634aa3833c09948d31d406 2024-11-14T02:45:39,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/e2f18a87f3634aa3833c09948d31d406, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T02:45:39,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/d4946b7a71e745239d06134c423653f5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/d4946b7a71e745239d06134c423653f5 2024-11-14T02:45:39,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/d4946b7a71e745239d06134c423653f5, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T02:45:39,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for ac0d8410ac14c280aa06089ab1aa8285 in 1289ms, sequenceid=12, compaction requested=false 2024-11-14T02:45:39,156 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T02:45:39,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:39,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T02:45:39,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:39,234 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-14T02:45:39,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:39,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:39,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:39,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:39,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:39,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:39,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/a501c50c2fcc41fea55dd86d6b61fee0 is 50, key is test_row_0/A:col10/1731552337932/Put/seqid=0 2024-11-14T02:45:39,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742220_1396 (size=12001) 2024-11-14T02:45:39,642 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/a501c50c2fcc41fea55dd86d6b61fee0 2024-11-14T02:45:39,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/6db8733203ac42909751d5dfd3b33df0 is 50, key is test_row_0/B:col10/1731552337932/Put/seqid=0 2024-11-14T02:45:39,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742221_1397 (size=12001) 2024-11-14T02:45:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T02:45:40,053 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/6db8733203ac42909751d5dfd3b33df0 2024-11-14T02:45:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/a506a593276e47cb9bcf03e7c90206cc is 50, key is test_row_0/C:col10/1731552337932/Put/seqid=0 2024-11-14T02:45:40,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742222_1398 (size=12001) 2024-11-14T02:45:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:40,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:40,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552400135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552400136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552400136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552400137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552400138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552400245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552400246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552400255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552400255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552400255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,376 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T02:45:40,463 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/a506a593276e47cb9bcf03e7c90206cc 2024-11-14T02:45:40,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552400455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552400455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552400475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552400475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552400475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/a501c50c2fcc41fea55dd86d6b61fee0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/a501c50c2fcc41fea55dd86d6b61fee0 2024-11-14T02:45:40,524 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/a501c50c2fcc41fea55dd86d6b61fee0, entries=150, sequenceid=46, filesize=11.7 K 2024-11-14T02:45:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/6db8733203ac42909751d5dfd3b33df0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/6db8733203ac42909751d5dfd3b33df0 2024-11-14T02:45:40,527 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/6db8733203ac42909751d5dfd3b33df0, entries=150, sequenceid=46, filesize=11.7 K 2024-11-14T02:45:40,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/a506a593276e47cb9bcf03e7c90206cc as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/a506a593276e47cb9bcf03e7c90206cc 2024-11-14T02:45:40,531 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/a506a593276e47cb9bcf03e7c90206cc, entries=150, sequenceid=46, filesize=11.7 K 2024-11-14T02:45:40,532 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for ac0d8410ac14c280aa06089ab1aa8285 in 1298ms, sequenceid=46, compaction requested=false 2024-11-14T02:45:40,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:40,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:40,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-14T02:45:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-14T02:45:40,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-14T02:45:40,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6750 sec 2024-11-14T02:45:40,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.6820 sec 2024-11-14T02:45:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:40,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T02:45:40,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:40,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:40,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:40,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:40,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:40,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:40,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/678ee9ed963d4f75a098b8b622cd8847 is 50, key is test_row_0/A:col10/1731552340135/Put/seqid=0 2024-11-14T02:45:40,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742223_1399 (size=16681) 2024-11-14T02:45:40,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552400804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552400805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552400813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552400814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552400815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552400925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552400925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552400925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552400935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552400936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:41,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552401135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:41,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552401135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:41,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552401135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:41,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552401145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:41,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552401155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:41,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/678ee9ed963d4f75a098b8b622cd8847 2024-11-14T02:45:41,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/ab8dad905e154f11bb66d58acf580d51 is 50, key is test_row_0/B:col10/1731552340135/Put/seqid=0 2024-11-14T02:45:41,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742224_1400 (size=12001) 2024-11-14T02:45:41,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552401445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552401445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552401445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:41,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552401455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:41,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:41,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552401465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:41,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/ab8dad905e154f11bb66d58acf580d51 2024-11-14T02:45:41,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/db82982188514c55a6bd1d141b5c4081 is 50, key is test_row_0/C:col10/1731552340135/Put/seqid=0 2024-11-14T02:45:41,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742225_1401 (size=12001) 2024-11-14T02:45:41,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552401955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:41,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552401955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:41,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552401965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:41,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552401975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:41,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:41,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552401985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:41,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:41,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:42,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/db82982188514c55a6bd1d141b5c4081 2024-11-14T02:45:42,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/678ee9ed963d4f75a098b8b622cd8847 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/678ee9ed963d4f75a098b8b622cd8847 2024-11-14T02:45:42,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/678ee9ed963d4f75a098b8b622cd8847, entries=250, sequenceid=59, filesize=16.3 K 2024-11-14T02:45:42,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/ab8dad905e154f11bb66d58acf580d51 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/ab8dad905e154f11bb66d58acf580d51 2024-11-14T02:45:42,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/ab8dad905e154f11bb66d58acf580d51, entries=150, sequenceid=59, filesize=11.7 K 2024-11-14T02:45:42,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/db82982188514c55a6bd1d141b5c4081 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/db82982188514c55a6bd1d141b5c4081 2024-11-14T02:45:42,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/db82982188514c55a6bd1d141b5c4081, entries=150, sequenceid=59, filesize=11.7 K 2024-11-14T02:45:42,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ac0d8410ac14c280aa06089ab1aa8285 in 1246ms, sequenceid=59, compaction requested=true 2024-11-14T02:45:42,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:42,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:42,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:42,023 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:42,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:42,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:42,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:42,023 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:42,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:42,024 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:42,024 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:42,024 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/A is initiating minor compaction (all files) 2024-11-14T02:45:42,024 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/B is initiating minor compaction (all files) 2024-11-14T02:45:42,024 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/B in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:42,024 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/A in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:42,024 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/e2f18a87f3634aa3833c09948d31d406, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/6db8733203ac42909751d5dfd3b33df0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/ab8dad905e154f11bb66d58acf580d51] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=35.2 K 2024-11-14T02:45:42,024 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/37d967b862454008ac2b41cea52b9f91, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/a501c50c2fcc41fea55dd86d6b61fee0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/678ee9ed963d4f75a098b8b622cd8847] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=44.3 K 2024-11-14T02:45:42,024 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting e2f18a87f3634aa3833c09948d31d406, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731552337863 2024-11-14T02:45:42,024 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37d967b862454008ac2b41cea52b9f91, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731552337863 2024-11-14T02:45:42,025 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 6db8733203ac42909751d5dfd3b33df0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731552337932 2024-11-14T02:45:42,025 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting a501c50c2fcc41fea55dd86d6b61fee0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731552337932 2024-11-14T02:45:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T02:45:42,025 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting ab8dad905e154f11bb66d58acf580d51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552340135 2024-11-14T02:45:42,025 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:42,025 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 678ee9ed963d4f75a098b8b622cd8847, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552340135 2024-11-14T02:45:42,026 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-14T02:45:42,028 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T02:45:42,028 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:42,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:42,036 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#B#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:42,037 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/e49ca53dbdec4b47af2563d698f45737 is 50, key is test_row_0/B:col10/1731552340135/Put/seqid=0 2024-11-14T02:45:42,038 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#A#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:42,039 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/afc258ed450c4094bf2dfd051e5c35f9 is 50, key is test_row_0/A:col10/1731552340135/Put/seqid=0 2024-11-14T02:45:42,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742226_1402 (size=12104) 2024-11-14T02:45:42,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742227_1403 (size=12104) 2024-11-14T02:45:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T02:45:42,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T02:45:42,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:42,181 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T02:45:42,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:42,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:42,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:42,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:42,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:42,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:42,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/3653f72cd8cf4df1ba0ba7e06ab235b8 is 50, key is test_row_0/A:col10/1731552340811/Put/seqid=0 2024-11-14T02:45:42,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742228_1404 (size=12001) 2024-11-14T02:45:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T02:45:42,451 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/e49ca53dbdec4b47af2563d698f45737 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/e49ca53dbdec4b47af2563d698f45737 2024-11-14T02:45:42,451 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/afc258ed450c4094bf2dfd051e5c35f9 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/afc258ed450c4094bf2dfd051e5c35f9 2024-11-14T02:45:42,455 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/B of ac0d8410ac14c280aa06089ab1aa8285 into e49ca53dbdec4b47af2563d698f45737(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:42,455 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/A of ac0d8410ac14c280aa06089ab1aa8285 into afc258ed450c4094bf2dfd051e5c35f9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:42,455 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:42,455 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:42,455 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/B, priority=13, startTime=1731552342023; duration=0sec 2024-11-14T02:45:42,455 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/A, priority=13, startTime=1731552342023; duration=0sec 2024-11-14T02:45:42,455 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:42,455 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:A 2024-11-14T02:45:42,455 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:42,455 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:B 2024-11-14T02:45:42,455 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:42,456 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:42,456 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/C is initiating minor compaction (all files) 2024-11-14T02:45:42,456 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/C in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:42,456 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/d4946b7a71e745239d06134c423653f5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/a506a593276e47cb9bcf03e7c90206cc, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/db82982188514c55a6bd1d141b5c4081] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=35.2 K 2024-11-14T02:45:42,457 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4946b7a71e745239d06134c423653f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731552337863 2024-11-14T02:45:42,457 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting a506a593276e47cb9bcf03e7c90206cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731552337932 2024-11-14T02:45:42,457 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting db82982188514c55a6bd1d141b5c4081, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552340135 2024-11-14T02:45:42,463 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#C#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:42,463 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/c46b15a11ffc4321a4fbe61c3352e758 is 50, key is test_row_0/C:col10/1731552340135/Put/seqid=0 2024-11-14T02:45:42,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742229_1405 (size=12104) 2024-11-14T02:45:42,589 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/3653f72cd8cf4df1ba0ba7e06ab235b8 2024-11-14T02:45:42,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/c11f4763d0294243a2145a39e58e087c is 50, key is test_row_0/B:col10/1731552340811/Put/seqid=0 2024-11-14T02:45:42,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742230_1406 (size=12001) 2024-11-14T02:45:42,607 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/c11f4763d0294243a2145a39e58e087c 2024-11-14T02:45:42,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/c50a4d93197a446f93326e709cf2f1ac is 50, key is test_row_0/C:col10/1731552340811/Put/seqid=0 2024-11-14T02:45:42,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742231_1407 (size=12001) 2024-11-14T02:45:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T02:45:42,873 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/c46b15a11ffc4321a4fbe61c3352e758 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c46b15a11ffc4321a4fbe61c3352e758 2024-11-14T02:45:42,877 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/C of ac0d8410ac14c280aa06089ab1aa8285 into c46b15a11ffc4321a4fbe61c3352e758(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:42,877 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:42,877 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/C, priority=13, startTime=1731552342023; duration=0sec 2024-11-14T02:45:42,877 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:42,877 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:C 2024-11-14T02:45:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:42,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:42,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552402991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:42,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:42,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:42,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552402994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552402994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552402995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,001 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in ac0d8410ac14c280aa06089ab1aa8285 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:45:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552403005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,011 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_0' in ac0d8410ac14c280aa06089ab1aa8285 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:45:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,021 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/c50a4d93197a446f93326e709cf2f1ac 2024-11-14T02:45:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/3653f72cd8cf4df1ba0ba7e06ab235b8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/3653f72cd8cf4df1ba0ba7e06ab235b8 2024-11-14T02:45:43,029 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/3653f72cd8cf4df1ba0ba7e06ab235b8, entries=150, sequenceid=78, filesize=11.7 K 2024-11-14T02:45:43,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/c11f4763d0294243a2145a39e58e087c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/c11f4763d0294243a2145a39e58e087c 2024-11-14T02:45:43,033 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/c11f4763d0294243a2145a39e58e087c, entries=150, sequenceid=78, filesize=11.7 K 2024-11-14T02:45:43,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/c50a4d93197a446f93326e709cf2f1ac as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c50a4d93197a446f93326e709cf2f1ac 2024-11-14T02:45:43,037 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c50a4d93197a446f93326e709cf2f1ac, entries=150, sequenceid=78, filesize=11.7 K 2024-11-14T02:45:43,038 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ac0d8410ac14c280aa06089ab1aa8285 in 857ms, sequenceid=78, compaction requested=false 2024-11-14T02:45:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-14T02:45:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-14T02:45:43,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-14T02:45:43,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0110 sec 2024-11-14T02:45:43,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.0170 sec 2024-11-14T02:45:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:43,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T02:45:43,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:43,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:43,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:43,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:43,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:43,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:43,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/7f81b79493b44ca4b7a1f09140981aad is 50, key is test_row_0/A:col10/1731552342990/Put/seqid=0 2024-11-14T02:45:43,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742232_1408 (size=11997) 2024-11-14T02:45:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T02:45:43,165 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:43,165 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-14T02:45:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T02:45:43,166 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:43,167 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:43,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:43,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552403175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552403175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552403182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T02:45:43,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552403285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552403285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552403295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T02:45:43,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:43,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:43,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:43,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T02:45:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:43,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T02:45:43,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552403495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552403505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552403505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/7f81b79493b44ca4b7a1f09140981aad 2024-11-14T02:45:43,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/49b192b607694c8c9990ce560fb0c6bd is 50, key is test_row_0/B:col10/1731552342990/Put/seqid=0 2024-11-14T02:45:43,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742233_1409 (size=9657) 2024-11-14T02:45:43,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T02:45:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:43,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:43,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T02:45:43,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:43,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:43,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:43,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:43,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T02:45:43,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552403815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552403816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552403825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:43,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:43,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:43,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/49b192b607694c8c9990ce560fb0c6bd 2024-11-14T02:45:43,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T02:45:43,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:43,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:43,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:43,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/3766fc66c38c415292099cd188c148d0 is 50, key is test_row_0/C:col10/1731552342990/Put/seqid=0 2024-11-14T02:45:43,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742234_1410 (size=9657) 2024-11-14T02:45:43,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/3766fc66c38c415292099cd188c148d0 2024-11-14T02:45:43,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/7f81b79493b44ca4b7a1f09140981aad as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/7f81b79493b44ca4b7a1f09140981aad 2024-11-14T02:45:43,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/7f81b79493b44ca4b7a1f09140981aad, entries=150, sequenceid=95, filesize=11.7 K 2024-11-14T02:45:43,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/49b192b607694c8c9990ce560fb0c6bd as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/49b192b607694c8c9990ce560fb0c6bd 2024-11-14T02:45:43,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/49b192b607694c8c9990ce560fb0c6bd, entries=100, sequenceid=95, filesize=9.4 K 2024-11-14T02:45:43,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/3766fc66c38c415292099cd188c148d0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3766fc66c38c415292099cd188c148d0 2024-11-14T02:45:43,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3766fc66c38c415292099cd188c148d0, entries=100, sequenceid=95, filesize=9.4 K 2024-11-14T02:45:43,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ac0d8410ac14c280aa06089ab1aa8285 in 851ms, sequenceid=95, compaction requested=true 2024-11-14T02:45:43,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:43,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:43,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:43,958 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:43,958 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:43,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:43,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:43,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:43,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:43,958 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36102 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:43,958 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:43,958 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/B is initiating minor compaction (all files) 2024-11-14T02:45:43,958 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/A is initiating minor compaction (all files) 2024-11-14T02:45:43,958 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/B in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:43,958 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/A in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:43,958 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/afc258ed450c4094bf2dfd051e5c35f9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/3653f72cd8cf4df1ba0ba7e06ab235b8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/7f81b79493b44ca4b7a1f09140981aad] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=35.3 K 2024-11-14T02:45:43,958 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/e49ca53dbdec4b47af2563d698f45737, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/c11f4763d0294243a2145a39e58e087c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/49b192b607694c8c9990ce560fb0c6bd] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=33.0 K 2024-11-14T02:45:43,959 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting e49ca53dbdec4b47af2563d698f45737, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552340135 2024-11-14T02:45:43,959 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting afc258ed450c4094bf2dfd051e5c35f9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552340135 2024-11-14T02:45:43,959 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting c11f4763d0294243a2145a39e58e087c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731552340804 2024-11-14T02:45:43,959 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3653f72cd8cf4df1ba0ba7e06ab235b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731552340804 2024-11-14T02:45:43,959 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 49b192b607694c8c9990ce560fb0c6bd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731552342990 2024-11-14T02:45:43,959 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f81b79493b44ca4b7a1f09140981aad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731552342990 2024-11-14T02:45:43,964 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#B#compaction#344 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:43,964 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/493b39127fa7449e8d510e70bfa59348 is 50, key is test_row_0/B:col10/1731552342990/Put/seqid=0 2024-11-14T02:45:43,966 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#A#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:43,967 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/9ad92f7286b84e839fa8ebdff4f56688 is 50, key is test_row_0/A:col10/1731552342990/Put/seqid=0 2024-11-14T02:45:43,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742235_1411 (size=12207) 2024-11-14T02:45:43,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742236_1412 (size=12207) 2024-11-14T02:45:43,973 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/9ad92f7286b84e839fa8ebdff4f56688 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/9ad92f7286b84e839fa8ebdff4f56688 2024-11-14T02:45:43,979 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/A of ac0d8410ac14c280aa06089ab1aa8285 into 9ad92f7286b84e839fa8ebdff4f56688(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:43,979 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:43,979 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/A, priority=13, startTime=1731552343957; duration=0sec 2024-11-14T02:45:43,979 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:43,979 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:A 2024-11-14T02:45:43,979 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:43,980 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:43,980 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/C is initiating minor compaction (all files) 2024-11-14T02:45:43,980 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/C in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:43,980 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c46b15a11ffc4321a4fbe61c3352e758, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c50a4d93197a446f93326e709cf2f1ac, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3766fc66c38c415292099cd188c148d0] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=33.0 K 2024-11-14T02:45:43,980 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting c46b15a11ffc4321a4fbe61c3352e758, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731552340135 2024-11-14T02:45:43,980 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting c50a4d93197a446f93326e709cf2f1ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731552340804 2024-11-14T02:45:43,981 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3766fc66c38c415292099cd188c148d0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731552342990 2024-11-14T02:45:43,986 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#C#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:43,987 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/ad12855e92174c249e1242f6b32a24d3 is 50, key is test_row_0/C:col10/1731552342990/Put/seqid=0 2024-11-14T02:45:43,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742237_1413 (size=12207) 2024-11-14T02:45:43,996 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/ad12855e92174c249e1242f6b32a24d3 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/ad12855e92174c249e1242f6b32a24d3 2024-11-14T02:45:44,001 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/C of ac0d8410ac14c280aa06089ab1aa8285 into ad12855e92174c249e1242f6b32a24d3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:44,001 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:44,001 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/C, priority=13, startTime=1731552343958; duration=0sec 2024-11-14T02:45:44,001 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:44,001 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:C 2024-11-14T02:45:44,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T02:45:44,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:44,081 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T02:45:44,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:44,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:44,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:44,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:44,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:44,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:44,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/3b8899cd55f3463289b62c08438719f1 is 50, key is test_row_0/A:col10/1731552343181/Put/seqid=0 2024-11-14T02:45:44,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742238_1414 (size=12001) 2024-11-14T02:45:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T02:45:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:44,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:44,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552404369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:44,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:44,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552404370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:44,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:44,373 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/493b39127fa7449e8d510e70bfa59348 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/493b39127fa7449e8d510e70bfa59348 2024-11-14T02:45:44,377 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/B of ac0d8410ac14c280aa06089ab1aa8285 into 493b39127fa7449e8d510e70bfa59348(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:44,377 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:44,377 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/B, priority=13, startTime=1731552343957; duration=0sec 2024-11-14T02:45:44,377 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:44,377 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:B 2024-11-14T02:45:44,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552404372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:44,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552404475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552404475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:44,489 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/3b8899cd55f3463289b62c08438719f1 2024-11-14T02:45:44,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/61abf24192a7419a989d9abc79b05416 is 50, key is test_row_0/B:col10/1731552343181/Put/seqid=0 2024-11-14T02:45:44,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742239_1415 (size=12001) 2024-11-14T02:45:44,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552404495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:44,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:44,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552404695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:44,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552404695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:44,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552404705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:44,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:44,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:44,899 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/61abf24192a7419a989d9abc79b05416 2024-11-14T02:45:44,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/cf007a24743d472cb43c87acaf78a962 is 50, key is test_row_0/C:col10/1731552343181/Put/seqid=0 2024-11-14T02:45:44,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742240_1416 (size=12001) 2024-11-14T02:45:45,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:45,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552405005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:45,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:45,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552405005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:45,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:45,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552405015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:45,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:45,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:45,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552405025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:45,028 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in ac0d8410ac14c280aa06089ab1aa8285 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:45:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:45,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:45,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552405026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:45,033 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_0' in ac0d8410ac14c280aa06089ab1aa8285 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:45:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:45,309 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/cf007a24743d472cb43c87acaf78a962 2024-11-14T02:45:45,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/3b8899cd55f3463289b62c08438719f1 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/3b8899cd55f3463289b62c08438719f1 2024-11-14T02:45:45,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T02:45:45,319 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/3b8899cd55f3463289b62c08438719f1, entries=150, sequenceid=115, filesize=11.7 K 2024-11-14T02:45:45,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/61abf24192a7419a989d9abc79b05416 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/61abf24192a7419a989d9abc79b05416 2024-11-14T02:45:45,323 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/61abf24192a7419a989d9abc79b05416, entries=150, sequenceid=115, filesize=11.7 K 2024-11-14T02:45:45,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/cf007a24743d472cb43c87acaf78a962 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/cf007a24743d472cb43c87acaf78a962 2024-11-14T02:45:45,327 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/cf007a24743d472cb43c87acaf78a962, entries=150, sequenceid=115, filesize=11.7 K 2024-11-14T02:45:45,328 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ac0d8410ac14c280aa06089ab1aa8285 in 1247ms, sequenceid=115, compaction requested=false 2024-11-14T02:45:45,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:45,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:45,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-14T02:45:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-14T02:45:45,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-14T02:45:45,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1620 sec 2024-11-14T02:45:45,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.1660 sec 2024-11-14T02:45:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:45,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T02:45:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:45,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:45,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/5a4f4fb0278042a68952d72d848071a2 is 50, key is test_row_0/A:col10/1731552344368/Put/seqid=0 2024-11-14T02:45:45,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742241_1417 (size=12101) 2024-11-14T02:45:45,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552405608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:45,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:45,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552405609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:45,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:45,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552405615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:45,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552405725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:45,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552405725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552405725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:45,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:45,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552405935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:45,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552405935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:45,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:45,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552405935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:45,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:45,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:45,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/5a4f4fb0278042a68952d72d848071a2 2024-11-14T02:45:45,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/45d0f06aeea349a99de89a500b191909 is 50, key is test_row_0/B:col10/1731552344368/Put/seqid=0 2024-11-14T02:45:45,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742242_1418 (size=12101) 2024-11-14T02:45:46,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552406245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:46,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552406255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:46,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552406255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:46,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:46,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/45d0f06aeea349a99de89a500b191909 2024-11-14T02:45:46,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/0db76944ca5f4bcda45611cf24cadde2 is 50, key is test_row_0/C:col10/1731552344368/Put/seqid=0 2024-11-14T02:45:46,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742243_1419 (size=12101) 2024-11-14T02:45:46,489 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T02:45:46,489 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T02:45:46,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552406755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:46,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552406775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:46,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:46,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:46,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552406775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:46,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/0db76944ca5f4bcda45611cf24cadde2 2024-11-14T02:45:46,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/5a4f4fb0278042a68952d72d848071a2 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/5a4f4fb0278042a68952d72d848071a2 2024-11-14T02:45:46,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/5a4f4fb0278042a68952d72d848071a2, entries=150, sequenceid=133, filesize=11.8 K 2024-11-14T02:45:46,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/45d0f06aeea349a99de89a500b191909 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/45d0f06aeea349a99de89a500b191909 2024-11-14T02:45:46,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/45d0f06aeea349a99de89a500b191909, entries=150, sequenceid=133, filesize=11.8 K 2024-11-14T02:45:46,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/0db76944ca5f4bcda45611cf24cadde2 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/0db76944ca5f4bcda45611cf24cadde2 2024-11-14T02:45:46,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/0db76944ca5f4bcda45611cf24cadde2, entries=150, sequenceid=133, filesize=11.8 K 2024-11-14T02:45:46,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ac0d8410ac14c280aa06089ab1aa8285 in 1249ms, sequenceid=133, compaction requested=true 2024-11-14T02:45:46,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:46,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:46,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:46,822 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:46,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:46,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:46,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:46,822 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:46,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:46,822 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:46,822 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:46,823 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/B is initiating minor compaction (all files) 2024-11-14T02:45:46,823 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/A is initiating minor compaction (all files) 2024-11-14T02:45:46,823 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/B in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:46,823 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/A in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:46,823 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/493b39127fa7449e8d510e70bfa59348, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/61abf24192a7419a989d9abc79b05416, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/45d0f06aeea349a99de89a500b191909] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=35.5 K 2024-11-14T02:45:46,823 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/9ad92f7286b84e839fa8ebdff4f56688, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/3b8899cd55f3463289b62c08438719f1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/5a4f4fb0278042a68952d72d848071a2] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=35.5 K 2024-11-14T02:45:46,823 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 493b39127fa7449e8d510e70bfa59348, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731552340813 2024-11-14T02:45:46,823 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ad92f7286b84e839fa8ebdff4f56688, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731552340813 2024-11-14T02:45:46,823 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 61abf24192a7419a989d9abc79b05416, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731552343173 2024-11-14T02:45:46,823 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b8899cd55f3463289b62c08438719f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731552343173 2024-11-14T02:45:46,823 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 45d0f06aeea349a99de89a500b191909, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731552344368 2024-11-14T02:45:46,823 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a4f4fb0278042a68952d72d848071a2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731552344368 2024-11-14T02:45:46,830 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#B#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:46,830 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/5e882e18e1d040acaa315e49a44d35cc is 50, key is test_row_0/B:col10/1731552344368/Put/seqid=0 2024-11-14T02:45:46,833 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#A#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:46,833 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/1c1b791c906d4a4c947449a16ace4dfb is 50, key is test_row_0/A:col10/1731552344368/Put/seqid=0 2024-11-14T02:45:46,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742244_1420 (size=12409) 2024-11-14T02:45:46,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742245_1421 (size=12409) 2024-11-14T02:45:47,252 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/5e882e18e1d040acaa315e49a44d35cc as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/5e882e18e1d040acaa315e49a44d35cc 2024-11-14T02:45:47,252 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/1c1b791c906d4a4c947449a16ace4dfb as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/1c1b791c906d4a4c947449a16ace4dfb 2024-11-14T02:45:47,255 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/B of ac0d8410ac14c280aa06089ab1aa8285 into 5e882e18e1d040acaa315e49a44d35cc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:47,255 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/A of ac0d8410ac14c280aa06089ab1aa8285 into 1c1b791c906d4a4c947449a16ace4dfb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:47,255 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:47,255 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:47,255 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/B, priority=13, startTime=1731552346822; duration=0sec 2024-11-14T02:45:47,255 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/A, priority=13, startTime=1731552346822; duration=0sec 2024-11-14T02:45:47,255 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:47,255 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:B 2024-11-14T02:45:47,255 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:47,255 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:A 2024-11-14T02:45:47,255 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:47,256 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:47,256 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/C is initiating minor compaction (all files) 2024-11-14T02:45:47,256 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/C in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:47,256 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/ad12855e92174c249e1242f6b32a24d3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/cf007a24743d472cb43c87acaf78a962, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/0db76944ca5f4bcda45611cf24cadde2] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=35.5 K 2024-11-14T02:45:47,257 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting ad12855e92174c249e1242f6b32a24d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731552340813 2024-11-14T02:45:47,257 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting cf007a24743d472cb43c87acaf78a962, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731552343173 2024-11-14T02:45:47,257 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 0db76944ca5f4bcda45611cf24cadde2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731552344368 2024-11-14T02:45:47,263 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#C#compaction#355 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:47,263 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/63f2c48fe53d46448a2c14c26009e5cf is 50, key is test_row_0/C:col10/1731552344368/Put/seqid=0 2024-11-14T02:45:47,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742246_1422 (size=12409) 2024-11-14T02:45:47,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T02:45:47,335 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:47,336 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:47,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-14T02:45:47,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T02:45:47,337 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:47,338 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:47,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T02:45:47,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T02:45:47,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:47,490 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T02:45:47,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:47,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:47,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:47,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:47,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:47,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/d9ce3193667a40cc875721eb6259c92a is 50, key is test_row_0/A:col10/1731552345613/Put/seqid=0 2024-11-14T02:45:47,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742247_1423 (size=12151) 2024-11-14T02:45:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T02:45:47,678 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/63f2c48fe53d46448a2c14c26009e5cf as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/63f2c48fe53d46448a2c14c26009e5cf 2024-11-14T02:45:47,682 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/C of ac0d8410ac14c280aa06089ab1aa8285 into 63f2c48fe53d46448a2c14c26009e5cf(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:47,682 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:47,683 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/C, priority=13, startTime=1731552346822; duration=0sec 2024-11-14T02:45:47,683 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:47,683 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:C 2024-11-14T02:45:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:47,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:47,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552407807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:47,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552407811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:47,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552407812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:47,901 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/d9ce3193667a40cc875721eb6259c92a 2024-11-14T02:45:47,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/66992749e2a14221b4edc7d0b6e44210 is 50, key is test_row_0/B:col10/1731552345613/Put/seqid=0 2024-11-14T02:45:47,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742248_1424 (size=12151) 2024-11-14T02:45:47,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552407915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:47,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552407925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:47,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552407926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T02:45:48,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552408125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:48,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552408135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:48,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552408135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:48,310 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/66992749e2a14221b4edc7d0b6e44210 2024-11-14T02:45:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/67f00eaae26b46a09cae66eff56d7ef1 is 50, key is test_row_0/C:col10/1731552345613/Put/seqid=0 2024-11-14T02:45:48,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742249_1425 (size=12151) 2024-11-14T02:45:48,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552408435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:48,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552408445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:48,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:48,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:48,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:48,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552408445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:48,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:48,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:48,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T02:45:48,720 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/67f00eaae26b46a09cae66eff56d7ef1 2024-11-14T02:45:48,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/d9ce3193667a40cc875721eb6259c92a as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/d9ce3193667a40cc875721eb6259c92a 2024-11-14T02:45:48,726 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/d9ce3193667a40cc875721eb6259c92a, entries=150, sequenceid=150, filesize=11.9 K 2024-11-14T02:45:48,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/66992749e2a14221b4edc7d0b6e44210 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/66992749e2a14221b4edc7d0b6e44210 2024-11-14T02:45:48,730 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/66992749e2a14221b4edc7d0b6e44210, entries=150, sequenceid=150, filesize=11.9 K 2024-11-14T02:45:48,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/67f00eaae26b46a09cae66eff56d7ef1 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/67f00eaae26b46a09cae66eff56d7ef1 2024-11-14T02:45:48,733 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/67f00eaae26b46a09cae66eff56d7ef1, entries=150, sequenceid=150, filesize=11.9 K 2024-11-14T02:45:48,734 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ac0d8410ac14c280aa06089ab1aa8285 in 1244ms, sequenceid=150, compaction requested=false 2024-11-14T02:45:48,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:48,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:48,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-14T02:45:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-14T02:45:48,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-14T02:45:48,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3970 sec 2024-11-14T02:45:48,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.4010 sec 2024-11-14T02:45:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:48,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T02:45:48,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:48,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:48,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:48,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:48,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:48,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:48,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/abc89cf6b6bc4f6c9b75d41a70b5f726 is 50, key is test_row_0/A:col10/1731552348946/Put/seqid=0 2024-11-14T02:45:48,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742250_1426 (size=14541) 2024-11-14T02:45:49,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552409007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:49,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552409008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:49,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552409009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:49,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57110 deadline: 1731552409045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:49,052 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in ac0d8410ac14c280aa06089ab1aa8285 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:45:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:49,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57094 deadline: 1731552409075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:49,083 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_0' in ac0d8410ac14c280aa06089ab1aa8285 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:45:49,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:49,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:49,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552409115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:49,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:49,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:49,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552409125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:49,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552409125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:49,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:49,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552409335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:49,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:49,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552409335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:49,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:49,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:49,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552409335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:49,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:49,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/abc89cf6b6bc4f6c9b75d41a70b5f726 2024-11-14T02:45:49,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/835f4d95efea4e96b85ed2d66c74a7a5 is 50, key is test_row_0/B:col10/1731552348946/Put/seqid=0 2024-11-14T02:45:49,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742251_1427 (size=12151) 2024-11-14T02:45:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T02:45:49,485 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:49,486 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-14T02:45:49,487 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T02:45:49,487 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:49,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T02:45:49,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T02:45:49,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:49,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:49,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:49,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:49,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:49,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552409655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552409655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:49,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552409655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:49,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:49,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/835f4d95efea4e96b85ed2d66c74a7a5 2024-11-14T02:45:49,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/8ecdb9ac2fbc4b8ebffb0dd78295af0f is 50, key is test_row_0/C:col10/1731552348946/Put/seqid=0 2024-11-14T02:45:49,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742252_1428 (size=12151) 2024-11-14T02:45:49,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T02:45:49,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:49,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:49,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:49,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:49,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T02:45:49,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T02:45:49,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:49,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:49,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:49,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:49,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:50,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T02:45:50,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:50,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:50,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:50,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:50,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T02:45:50,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/8ecdb9ac2fbc4b8ebffb0dd78295af0f 2024-11-14T02:45:50,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/abc89cf6b6bc4f6c9b75d41a70b5f726 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/abc89cf6b6bc4f6c9b75d41a70b5f726 2024-11-14T02:45:50,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/abc89cf6b6bc4f6c9b75d41a70b5f726, entries=200, sequenceid=169, filesize=14.2 K 2024-11-14T02:45:50,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/835f4d95efea4e96b85ed2d66c74a7a5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/835f4d95efea4e96b85ed2d66c74a7a5 2024-11-14T02:45:50,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552410175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:50,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:50,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:50,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552410175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:50,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:50,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:50,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:50,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552410175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:50,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:50,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:50,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:50,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:50,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/835f4d95efea4e96b85ed2d66c74a7a5, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T02:45:50,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/8ecdb9ac2fbc4b8ebffb0dd78295af0f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/8ecdb9ac2fbc4b8ebffb0dd78295af0f 2024-11-14T02:45:50,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/8ecdb9ac2fbc4b8ebffb0dd78295af0f, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T02:45:50,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ac0d8410ac14c280aa06089ab1aa8285 in 1240ms, sequenceid=169, compaction requested=true 2024-11-14T02:45:50,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:50,187 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:50,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:50,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:50,188 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:50,188 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:50,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:50,188 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/A is initiating minor compaction (all files) 2024-11-14T02:45:50,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:50,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:50,188 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/A in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:50,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:50,188 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/1c1b791c906d4a4c947449a16ace4dfb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/d9ce3193667a40cc875721eb6259c92a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/abc89cf6b6bc4f6c9b75d41a70b5f726] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=38.2 K 2024-11-14T02:45:50,188 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c1b791c906d4a4c947449a16ace4dfb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731552344368 2024-11-14T02:45:50,189 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9ce3193667a40cc875721eb6259c92a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731552345607 2024-11-14T02:45:50,189 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:50,189 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting abc89cf6b6bc4f6c9b75d41a70b5f726, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731552347800 2024-11-14T02:45:50,189 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/B is initiating minor compaction (all files) 2024-11-14T02:45:50,189 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/B in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:50,189 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/5e882e18e1d040acaa315e49a44d35cc, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/66992749e2a14221b4edc7d0b6e44210, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/835f4d95efea4e96b85ed2d66c74a7a5] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=35.9 K 2024-11-14T02:45:50,189 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e882e18e1d040acaa315e49a44d35cc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731552344368 2024-11-14T02:45:50,190 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 66992749e2a14221b4edc7d0b6e44210, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731552345607 2024-11-14T02:45:50,190 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 835f4d95efea4e96b85ed2d66c74a7a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731552347800 2024-11-14T02:45:50,197 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#A#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:50,197 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/0c8854924b0149ce9c7e4a623d2d0b53 is 50, key is test_row_0/A:col10/1731552348946/Put/seqid=0 2024-11-14T02:45:50,200 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#B#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:50,201 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/cae6ec144cf1432dad2cf8d5985d400b is 50, key is test_row_0/B:col10/1731552348946/Put/seqid=0 2024-11-14T02:45:50,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742253_1429 (size=12561) 2024-11-14T02:45:50,208 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/0c8854924b0149ce9c7e4a623d2d0b53 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/0c8854924b0149ce9c7e4a623d2d0b53 2024-11-14T02:45:50,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742254_1430 (size=12561) 2024-11-14T02:45:50,219 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/cae6ec144cf1432dad2cf8d5985d400b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/cae6ec144cf1432dad2cf8d5985d400b 2024-11-14T02:45:50,221 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/A of ac0d8410ac14c280aa06089ab1aa8285 into 0c8854924b0149ce9c7e4a623d2d0b53(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:50,221 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:50,221 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/A, priority=13, startTime=1731552350187; duration=0sec 2024-11-14T02:45:50,221 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:50,221 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:A 2024-11-14T02:45:50,221 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:50,222 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:50,222 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/C is initiating minor compaction (all files) 2024-11-14T02:45:50,222 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/C in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:50,222 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/63f2c48fe53d46448a2c14c26009e5cf, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/67f00eaae26b46a09cae66eff56d7ef1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/8ecdb9ac2fbc4b8ebffb0dd78295af0f] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=35.9 K 2024-11-14T02:45:50,222 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63f2c48fe53d46448a2c14c26009e5cf, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731552344368 2024-11-14T02:45:50,224 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67f00eaae26b46a09cae66eff56d7ef1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731552345607 2024-11-14T02:45:50,225 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ecdb9ac2fbc4b8ebffb0dd78295af0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731552347800 2024-11-14T02:45:50,226 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/B of ac0d8410ac14c280aa06089ab1aa8285 into cae6ec144cf1432dad2cf8d5985d400b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:50,226 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:50,226 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/B, priority=13, startTime=1731552350188; duration=0sec 2024-11-14T02:45:50,226 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:50,226 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:B 2024-11-14T02:45:50,236 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#C#compaction#364 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:50,236 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/56faed4dbdb8446990d05ca1cf4b9409 is 50, key is test_row_0/C:col10/1731552348946/Put/seqid=0 2024-11-14T02:45:50,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742255_1431 (size=12561) 2024-11-14T02:45:50,249 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/56faed4dbdb8446990d05ca1cf4b9409 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/56faed4dbdb8446990d05ca1cf4b9409 2024-11-14T02:45:50,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T02:45:50,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:50,250 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T02:45:50,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:50,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:50,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:50,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:50,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:50,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:50,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/7023025de17f47818e2eb157b49e06e9 is 50, key is test_row_0/A:col10/1731552349008/Put/seqid=0 2024-11-14T02:45:50,260 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/C of ac0d8410ac14c280aa06089ab1aa8285 into 56faed4dbdb8446990d05ca1cf4b9409(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:50,260 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:50,260 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/C, priority=13, startTime=1731552350188; duration=0sec 2024-11-14T02:45:50,260 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:50,260 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:C 2024-11-14T02:45:50,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742256_1432 (size=12151) 2024-11-14T02:45:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T02:45:50,665 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/7023025de17f47818e2eb157b49e06e9 2024-11-14T02:45:50,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/8dbde70824e54ea8aeb1e8a8ea283b5c is 50, key is test_row_0/B:col10/1731552349008/Put/seqid=0 2024-11-14T02:45:50,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742257_1433 (size=12151) 2024-11-14T02:45:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:51,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:51,190 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/8dbde70824e54ea8aeb1e8a8ea283b5c 2024-11-14T02:45:51,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/3f6887f92af242e29e8512e85981de41 is 50, key is test_row_0/C:col10/1731552349008/Put/seqid=0 2024-11-14T02:45:51,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742258_1434 (size=12151) 2024-11-14T02:45:51,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552411229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:51,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:51,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552411230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552411233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:51,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552411345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:51,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552411345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552411345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:51,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552411555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:51,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552411555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:51,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552411555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:51,599 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/3f6887f92af242e29e8512e85981de41 2024-11-14T02:45:51,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/7023025de17f47818e2eb157b49e06e9 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/7023025de17f47818e2eb157b49e06e9 2024-11-14T02:45:51,605 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/7023025de17f47818e2eb157b49e06e9, entries=150, sequenceid=185, filesize=11.9 K 2024-11-14T02:45:51,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/8dbde70824e54ea8aeb1e8a8ea283b5c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/8dbde70824e54ea8aeb1e8a8ea283b5c 2024-11-14T02:45:51,611 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/8dbde70824e54ea8aeb1e8a8ea283b5c, entries=150, sequenceid=185, filesize=11.9 K 2024-11-14T02:45:51,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/3f6887f92af242e29e8512e85981de41 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3f6887f92af242e29e8512e85981de41 2024-11-14T02:45:51,615 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3f6887f92af242e29e8512e85981de41, entries=150, sequenceid=185, filesize=11.9 K 2024-11-14T02:45:51,617 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ac0d8410ac14c280aa06089ab1aa8285 in 1367ms, sequenceid=185, compaction requested=false 2024-11-14T02:45:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-14T02:45:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-14T02:45:51,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-14T02:45:51,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1320 sec 2024-11-14T02:45:51,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.1370 sec 2024-11-14T02:45:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T02:45:51,645 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:51,646 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-14T02:45:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T02:45:51,647 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:51,648 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:51,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T02:45:51,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T02:45:51,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:51,800 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T02:45:51,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:51,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:51,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:51,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:51,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:51,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:51,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/ac0bcc62675a40aa8b176595a4e8143a is 50, key is test_row_0/A:col10/1731552351229/Put/seqid=0 2024-11-14T02:45:51,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742259_1435 (size=12151) 2024-11-14T02:45:51,811 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/ac0bcc62675a40aa8b176595a4e8143a 2024-11-14T02:45:51,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/3089f28c2222434788d278fa7dbcc430 is 50, key is test_row_0/B:col10/1731552351229/Put/seqid=0 2024-11-14T02:45:51,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742260_1436 (size=12151) 2024-11-14T02:45:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:51,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:51,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552411892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:51,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552411893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:51,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552411896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:51,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:51,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T02:45:52,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552412005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:52,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552412005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:52,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:52,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552412015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:52,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552412216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:52,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:52,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552412216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:52,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:52,236 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/3089f28c2222434788d278fa7dbcc430 2024-11-14T02:45:52,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552412235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:52,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/cc3584aca4ff4bdc93cf9983f7af41e8 is 50, key is test_row_0/C:col10/1731552351229/Put/seqid=0 2024-11-14T02:45:52,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742261_1437 (size=12151) 2024-11-14T02:45:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T02:45:52,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552412525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:52,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552412525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:52,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552412545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:52,646 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/cc3584aca4ff4bdc93cf9983f7af41e8 2024-11-14T02:45:52,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/ac0bcc62675a40aa8b176595a4e8143a as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/ac0bcc62675a40aa8b176595a4e8143a 2024-11-14T02:45:52,653 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/ac0bcc62675a40aa8b176595a4e8143a, entries=150, sequenceid=204, filesize=11.9 K 2024-11-14T02:45:52,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/3089f28c2222434788d278fa7dbcc430 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/3089f28c2222434788d278fa7dbcc430 2024-11-14T02:45:52,657 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/3089f28c2222434788d278fa7dbcc430, entries=150, sequenceid=204, filesize=11.9 K 2024-11-14T02:45:52,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/cc3584aca4ff4bdc93cf9983f7af41e8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/cc3584aca4ff4bdc93cf9983f7af41e8 2024-11-14T02:45:52,660 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/cc3584aca4ff4bdc93cf9983f7af41e8, entries=150, sequenceid=204, filesize=11.9 K 2024-11-14T02:45:52,661 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ac0d8410ac14c280aa06089ab1aa8285 in 861ms, sequenceid=204, compaction requested=true 2024-11-14T02:45:52,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:52,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:52,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-14T02:45:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-14T02:45:52,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-14T02:45:52,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0140 sec 2024-11-14T02:45:52,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.0180 sec 2024-11-14T02:45:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T02:45:52,785 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:52,785 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-14T02:45:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T02:45:52,787 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:52,787 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:52,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T02:45:52,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T02:45:52,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:52,939 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T02:45:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:52,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/da6715c449b34f4d899df575eacde034 is 50, key is test_row_0/A:col10/1731552351892/Put/seqid=0 2024-11-14T02:45:52,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742262_1438 (size=12151) 2024-11-14T02:45:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:53,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:53,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552413093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:53,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552413093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:53,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T02:45:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552413097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:53,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:53,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552413205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:53,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552413205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:53,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552413215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:53,348 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/da6715c449b34f4d899df575eacde034 2024-11-14T02:45:53,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/50ff6ba533a241779a452870da8dc0bf is 50, key is test_row_0/B:col10/1731552351892/Put/seqid=0 2024-11-14T02:45:53,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742263_1439 (size=12151) 2024-11-14T02:45:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T02:45:53,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552413425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552413425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:53,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552413425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:53,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552413735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:53,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552413735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552413735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:53,757 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/50ff6ba533a241779a452870da8dc0bf 2024-11-14T02:45:53,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/fa4ac9d354d34f64b71d8e3769f6cb86 is 50, key is test_row_0/C:col10/1731552351892/Put/seqid=0 2024-11-14T02:45:53,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742264_1440 (size=12151) 2024-11-14T02:45:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T02:45:54,167 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/fa4ac9d354d34f64b71d8e3769f6cb86 2024-11-14T02:45:54,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/da6715c449b34f4d899df575eacde034 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/da6715c449b34f4d899df575eacde034 2024-11-14T02:45:54,173 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/da6715c449b34f4d899df575eacde034, entries=150, sequenceid=218, filesize=11.9 K 2024-11-14T02:45:54,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/50ff6ba533a241779a452870da8dc0bf as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/50ff6ba533a241779a452870da8dc0bf 2024-11-14T02:45:54,176 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/50ff6ba533a241779a452870da8dc0bf, entries=150, sequenceid=218, filesize=11.9 K 2024-11-14T02:45:54,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/fa4ac9d354d34f64b71d8e3769f6cb86 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/fa4ac9d354d34f64b71d8e3769f6cb86 2024-11-14T02:45:54,179 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/fa4ac9d354d34f64b71d8e3769f6cb86, entries=150, sequenceid=218, filesize=11.9 K 2024-11-14T02:45:54,180 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ac0d8410ac14c280aa06089ab1aa8285 in 1241ms, sequenceid=218, compaction requested=true 2024-11-14T02:45:54,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:54,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:54,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-14T02:45:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-14T02:45:54,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-14T02:45:54,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3940 sec 2024-11-14T02:45:54,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.3980 sec 2024-11-14T02:45:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:54,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T02:45:54,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:54,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:54,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:54,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:54,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:54,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:54,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/9b1b5a1cf62e4c528c8eec4e5c1c3509 is 50, key is test_row_0/A:col10/1731552354251/Put/seqid=0 2024-11-14T02:45:54,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742265_1441 (size=14541) 2024-11-14T02:45:54,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552414278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:54,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:54,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552414279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:54,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552414282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:54,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552414385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:54,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:54,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552414395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:54,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:54,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552414395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:54,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:54,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:54,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552414605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552414605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:54,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552414605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:54,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/9b1b5a1cf62e4c528c8eec4e5c1c3509 2024-11-14T02:45:54,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/66cc5cca38774162b7d5111ba91cfcab is 50, key is test_row_0/B:col10/1731552354251/Put/seqid=0 2024-11-14T02:45:54,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742266_1442 (size=12151) 2024-11-14T02:45:54,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552414925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552414925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552414925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T02:45:54,936 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:54,936 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-14T02:45:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T02:45:54,938 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:54,938 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:54,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T02:45:55,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/66cc5cca38774162b7d5111ba91cfcab 2024-11-14T02:45:55,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/ec89236cbcdb4a01b1aaff0218cf9c71 is 50, key is test_row_0/C:col10/1731552354251/Put/seqid=0 2024-11-14T02:45:55,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742267_1443 (size=12151) 2024-11-14T02:45:55,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T02:45:55,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:55,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:55,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:55,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,195 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T02:45:55,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T02:45:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:55,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T02:45:55,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T02:45:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:55,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:55,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552415435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:55,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552415445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552415445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:55,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/ec89236cbcdb4a01b1aaff0218cf9c71 2024-11-14T02:45:55,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/9b1b5a1cf62e4c528c8eec4e5c1c3509 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/9b1b5a1cf62e4c528c8eec4e5c1c3509 2024-11-14T02:45:55,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/9b1b5a1cf62e4c528c8eec4e5c1c3509, entries=200, sequenceid=237, filesize=14.2 K 2024-11-14T02:45:55,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/66cc5cca38774162b7d5111ba91cfcab as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/66cc5cca38774162b7d5111ba91cfcab 2024-11-14T02:45:55,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/66cc5cca38774162b7d5111ba91cfcab, entries=150, sequenceid=237, filesize=11.9 K 2024-11-14T02:45:55,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/ec89236cbcdb4a01b1aaff0218cf9c71 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/ec89236cbcdb4a01b1aaff0218cf9c71 2024-11-14T02:45:55,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/ec89236cbcdb4a01b1aaff0218cf9c71, entries=150, sequenceid=237, filesize=11.9 K 2024-11-14T02:45:55,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ac0d8410ac14c280aa06089ab1aa8285 in 1244ms, sequenceid=237, compaction requested=true 2024-11-14T02:45:55,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:55,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:55,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:55,496 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T02:45:55,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:55,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:55,496 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T02:45:55,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:55,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:55,498 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61165 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T02:45:55,498 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/B is initiating minor compaction (all files) 2024-11-14T02:45:55,498 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/B in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:55,498 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/cae6ec144cf1432dad2cf8d5985d400b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/8dbde70824e54ea8aeb1e8a8ea283b5c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/3089f28c2222434788d278fa7dbcc430, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/50ff6ba533a241779a452870da8dc0bf, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/66cc5cca38774162b7d5111ba91cfcab] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=59.7 K 2024-11-14T02:45:55,498 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 63555 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T02:45:55,498 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/A is initiating minor compaction (all files) 2024-11-14T02:45:55,498 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/A in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:55,498 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/0c8854924b0149ce9c7e4a623d2d0b53, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/7023025de17f47818e2eb157b49e06e9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/ac0bcc62675a40aa8b176595a4e8143a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/da6715c449b34f4d899df575eacde034, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/9b1b5a1cf62e4c528c8eec4e5c1c3509] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=62.1 K 2024-11-14T02:45:55,499 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c8854924b0149ce9c7e4a623d2d0b53, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731552347800 2024-11-14T02:45:55,499 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting cae6ec144cf1432dad2cf8d5985d400b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731552347800 2024-11-14T02:45:55,499 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7023025de17f47818e2eb157b49e06e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731552349006 2024-11-14T02:45:55,499 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dbde70824e54ea8aeb1e8a8ea283b5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731552349006 2024-11-14T02:45:55,499 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac0bcc62675a40aa8b176595a4e8143a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731552351219 2024-11-14T02:45:55,499 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 3089f28c2222434788d278fa7dbcc430, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731552351219 2024-11-14T02:45:55,499 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting da6715c449b34f4d899df575eacde034, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731552351879 2024-11-14T02:45:55,499 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 50ff6ba533a241779a452870da8dc0bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731552351879 2024-11-14T02:45:55,499 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b1b5a1cf62e4c528c8eec4e5c1c3509, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731552353083 2024-11-14T02:45:55,500 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 66cc5cca38774162b7d5111ba91cfcab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731552353083 2024-11-14T02:45:55,512 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#A#compaction#377 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:55,512 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/22dd0e2572e24af2b9e4ee190c39a79a is 50, key is test_row_0/A:col10/1731552354251/Put/seqid=0 2024-11-14T02:45:55,515 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#B#compaction#378 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:55,515 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/cc7da23b01a84899a8ce196bc623d0cb is 50, key is test_row_0/B:col10/1731552354251/Put/seqid=0 2024-11-14T02:45:55,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742269_1445 (size=12731) 2024-11-14T02:45:55,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742268_1444 (size=12731) 2024-11-14T02:45:55,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T02:45:55,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:55,548 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T02:45:55,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:55,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:55,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:55,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:55,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:55,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:55,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/fa43d266e0ac4b47b6657b9236a47824 is 50, key is test_row_0/A:col10/1731552354268/Put/seqid=0 2024-11-14T02:45:55,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742270_1446 (size=12151) 2024-11-14T02:45:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T02:45:55,981 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/fa43d266e0ac4b47b6657b9236a47824 2024-11-14T02:45:55,984 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/cc7da23b01a84899a8ce196bc623d0cb as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/cc7da23b01a84899a8ce196bc623d0cb 2024-11-14T02:45:55,984 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/22dd0e2572e24af2b9e4ee190c39a79a as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/22dd0e2572e24af2b9e4ee190c39a79a 2024-11-14T02:45:55,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/6cca875bfcd54c6fbd37a5078df55e8c is 50, key is test_row_0/B:col10/1731552354268/Put/seqid=0 2024-11-14T02:45:55,992 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/A of ac0d8410ac14c280aa06089ab1aa8285 into 22dd0e2572e24af2b9e4ee190c39a79a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:55,992 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/B of ac0d8410ac14c280aa06089ab1aa8285 into cc7da23b01a84899a8ce196bc623d0cb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:55,992 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:55,992 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:55,992 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/A, priority=11, startTime=1731552355496; duration=0sec 2024-11-14T02:45:55,992 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/B, priority=11, startTime=1731552355496; duration=0sec 2024-11-14T02:45:55,992 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:55,992 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:B 2024-11-14T02:45:55,992 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T02:45:55,992 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:55,993 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:A 2024-11-14T02:45:55,994 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61165 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T02:45:55,994 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/C is initiating minor compaction (all files) 2024-11-14T02:45:55,994 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/C in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:55,994 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/56faed4dbdb8446990d05ca1cf4b9409, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3f6887f92af242e29e8512e85981de41, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/cc3584aca4ff4bdc93cf9983f7af41e8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/fa4ac9d354d34f64b71d8e3769f6cb86, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/ec89236cbcdb4a01b1aaff0218cf9c71] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=59.7 K 2024-11-14T02:45:55,995 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 56faed4dbdb8446990d05ca1cf4b9409, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731552347800 2024-11-14T02:45:55,995 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f6887f92af242e29e8512e85981de41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731552349006 2024-11-14T02:45:55,998 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting cc3584aca4ff4bdc93cf9983f7af41e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731552351219 2024-11-14T02:45:55,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742271_1447 (size=12151) 2024-11-14T02:45:55,999 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting fa4ac9d354d34f64b71d8e3769f6cb86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731552351879 2024-11-14T02:45:55,999 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/6cca875bfcd54c6fbd37a5078df55e8c 2024-11-14T02:45:56,000 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting ec89236cbcdb4a01b1aaff0218cf9c71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731552353083 2024-11-14T02:45:56,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/c8d6a40369c94f82b499a9106ce3a2bd is 50, key is test_row_0/C:col10/1731552354268/Put/seqid=0 2024-11-14T02:45:56,010 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#C#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:56,010 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/148454cea4b04406a5c993bc157db3dd is 50, key is test_row_0/C:col10/1731552354251/Put/seqid=0 2024-11-14T02:45:56,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742272_1448 (size=12731) 2024-11-14T02:45:56,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742273_1449 (size=12151) 2024-11-14T02:45:56,021 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/c8d6a40369c94f82b499a9106ce3a2bd 2024-11-14T02:45:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/fa43d266e0ac4b47b6657b9236a47824 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/fa43d266e0ac4b47b6657b9236a47824 2024-11-14T02:45:56,027 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/fa43d266e0ac4b47b6657b9236a47824, entries=150, sequenceid=250, filesize=11.9 K 2024-11-14T02:45:56,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/6cca875bfcd54c6fbd37a5078df55e8c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/6cca875bfcd54c6fbd37a5078df55e8c 2024-11-14T02:45:56,030 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/6cca875bfcd54c6fbd37a5078df55e8c, entries=150, sequenceid=250, filesize=11.9 K 2024-11-14T02:45:56,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/c8d6a40369c94f82b499a9106ce3a2bd as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c8d6a40369c94f82b499a9106ce3a2bd 2024-11-14T02:45:56,034 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c8d6a40369c94f82b499a9106ce3a2bd, entries=150, sequenceid=250, filesize=11.9 K 2024-11-14T02:45:56,035 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for ac0d8410ac14c280aa06089ab1aa8285 in 487ms, sequenceid=250, compaction requested=false 2024-11-14T02:45:56,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:56,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:56,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-14T02:45:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-14T02:45:56,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-14T02:45:56,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0980 sec 2024-11-14T02:45:56,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.1020 sec 2024-11-14T02:45:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T02:45:56,075 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:56,076 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-14T02:45:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T02:45:56,077 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:56,077 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:56,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T02:45:56,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T02:45:56,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:56,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:56,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:56,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-14T02:45:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-14T02:45:56,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-14T02:45:56,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 153 msec 2024-11-14T02:45:56,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 157 msec 2024-11-14T02:45:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T02:45:56,396 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:45:56,396 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:45:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-14T02:45:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T02:45:56,400 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:45:56,400 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:45:56,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:45:56,421 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/148454cea4b04406a5c993bc157db3dd as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/148454cea4b04406a5c993bc157db3dd 2024-11-14T02:45:56,425 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/C of ac0d8410ac14c280aa06089ab1aa8285 into 148454cea4b04406a5c993bc157db3dd(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:56,426 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:56,426 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/C, priority=11, startTime=1731552355496; duration=0sec 2024-11-14T02:45:56,426 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:56,426 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:C 2024-11-14T02:45:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:56,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:45:56,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:56,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:56,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:56,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:56,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:56,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:56,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/58cd6d872aca424d91ca0f3cfee5b8e7 is 50, key is test_row_0/A:col10/1731552356465/Put/seqid=0 2024-11-14T02:45:56,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742274_1450 (size=17181) 2024-11-14T02:45:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T02:45:56,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731552356496, max timestamp=1731552356527Num uniques 859; 2024-11-14T02:45:56,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552356496, max timestamp=1731552356527Num uniques 859; 2024-11-14T02:45:56,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731552356496, max timestamp=1731552356527Num uniques 859; 2024-11-14T02:45:56,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T02:45:56,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:45:56,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T02:45:56,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:56,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:56,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:56,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:56,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:56,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:56,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552416581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:56,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552416584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:56,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552416586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:56,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552416695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:56,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:56,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:56,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:56,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:56,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:56,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:56,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T02:45:56,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552416705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:56,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552416705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:56,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:56,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:56,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:56,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:56,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:56,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:56,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/58cd6d872aca424d91ca0f3cfee5b8e7 2024-11-14T02:45:56,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/1c9fba41601f4a61841e712ef124c53f is 50, key is test_row_0/B:col10/1731552356465/Put/seqid=0 2024-11-14T02:45:56,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742275_1451 (size=12301) 2024-11-14T02:45:56,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552416905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:56,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:56,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552416925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:56,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552416925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:57,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:57,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:57,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T02:45:57,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:57,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:57,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552417215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:57,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:57,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552417245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:57,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:57,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552417245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:57,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:57,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/1c9fba41601f4a61841e712ef124c53f 2024-11-14T02:45:57,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/b2413d0796cd4d57993afcfc1d98305c is 50, key is test_row_0/C:col10/1731552356465/Put/seqid=0 2024-11-14T02:45:57,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742276_1452 (size=12301) 2024-11-14T02:45:57,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:57,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:57,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:57,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:57,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T02:45:57,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/b2413d0796cd4d57993afcfc1d98305c 2024-11-14T02:45:57,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/58cd6d872aca424d91ca0f3cfee5b8e7 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/58cd6d872aca424d91ca0f3cfee5b8e7 2024-11-14T02:45:57,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/58cd6d872aca424d91ca0f3cfee5b8e7, entries=250, sequenceid=264, filesize=16.8 K 2024-11-14T02:45:57,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/1c9fba41601f4a61841e712ef124c53f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/1c9fba41601f4a61841e712ef124c53f 2024-11-14T02:45:57,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/1c9fba41601f4a61841e712ef124c53f, entries=150, sequenceid=264, filesize=12.0 K 2024-11-14T02:45:57,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/b2413d0796cd4d57993afcfc1d98305c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/b2413d0796cd4d57993afcfc1d98305c 2024-11-14T02:45:57,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/b2413d0796cd4d57993afcfc1d98305c, entries=150, sequenceid=264, filesize=12.0 K 2024-11-14T02:45:57,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ac0d8410ac14c280aa06089ab1aa8285 in 1245ms, sequenceid=264, compaction requested=true 2024-11-14T02:45:57,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:57,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:45:57,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:57,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:45:57,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:57,712 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:57,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:45:57,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:57,712 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:57,713 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42063 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:57,713 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:57,713 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/A is initiating minor compaction (all files) 2024-11-14T02:45:57,713 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/B is initiating minor compaction (all files) 2024-11-14T02:45:57,713 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/A in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,713 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/B in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,713 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/22dd0e2572e24af2b9e4ee190c39a79a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/fa43d266e0ac4b47b6657b9236a47824, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/58cd6d872aca424d91ca0f3cfee5b8e7] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=41.1 K 2024-11-14T02:45:57,713 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/cc7da23b01a84899a8ce196bc623d0cb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/6cca875bfcd54c6fbd37a5078df55e8c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/1c9fba41601f4a61841e712ef124c53f] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=36.3 K 2024-11-14T02:45:57,714 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 22dd0e2572e24af2b9e4ee190c39a79a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731552353083 2024-11-14T02:45:57,714 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting fa43d266e0ac4b47b6657b9236a47824, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731552354268 2024-11-14T02:45:57,714 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc7da23b01a84899a8ce196bc623d0cb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731552353083 2024-11-14T02:45:57,714 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 58cd6d872aca424d91ca0f3cfee5b8e7, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731552356458 2024-11-14T02:45:57,714 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cca875bfcd54c6fbd37a5078df55e8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731552354268 2024-11-14T02:45:57,715 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c9fba41601f4a61841e712ef124c53f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731552356463 2024-11-14T02:45:57,720 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#B#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:57,720 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/a582900796824bcfa8359198fb4b8029 is 50, key is test_row_0/B:col10/1731552356465/Put/seqid=0 2024-11-14T02:45:57,725 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:57,725 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/2a1fc1cce69f4416aed60244320dddfb is 50, key is test_row_0/A:col10/1731552356465/Put/seqid=0 2024-11-14T02:45:57,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742277_1453 (size=12983) 2024-11-14T02:45:57,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742278_1454 (size=12983) 2024-11-14T02:45:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:57,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-14T02:45:57,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:57,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:57,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:57,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:57,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:57,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:45:57,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/14fa66a85ce047e8ab912225a19f08c1 is 50, key is test_row_0/A:col10/1731552356584/Put/seqid=0 2024-11-14T02:45:57,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742279_1455 (size=14741) 2024-11-14T02:45:57,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:57,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:57,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552417770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:57,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:57,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552417771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:57,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:57,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:57,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552417774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:57,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:57,856 INFO [Thread-1800 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1800. 2024-11-14T02:45:57,856 DEBUG [Thread-1800 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:57,857 DEBUG [Thread-1800 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:57,857 DEBUG [Thread-1800 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:57,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:57,867 INFO [Thread-1804 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1804. 2024-11-14T02:45:57,867 DEBUG [Thread-1804 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:57,867 DEBUG [Thread-1804 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:57,867 DEBUG [Thread-1804 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:57,867 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:57,868 INFO [Thread-1806 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1806. 2024-11-14T02:45:57,868 DEBUG [Thread-1806 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:57,868 DEBUG [Thread-1806 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:57,868 DEBUG [Thread-1806 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:57,868 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:57,868 INFO [Thread-1802 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1802. 2024-11-14T02:45:57,869 DEBUG [Thread-1802 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:57,869 DEBUG [Thread-1802 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:57,869 DEBUG [Thread-1802 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:57,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:57,882 INFO [Thread-1798 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1798. 2024-11-14T02:45:57,882 DEBUG [Thread-1798 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:57,882 DEBUG [Thread-1798 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:57,882 DEBUG [Thread-1798 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:57,882 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:57,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:57,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552417885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552417885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:57,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552417895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:57,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:57,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:58,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:58,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:58,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552418096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:58,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552418096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:58,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552418105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:58,138 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/a582900796824bcfa8359198fb4b8029 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/a582900796824bcfa8359198fb4b8029 2024-11-14T02:45:58,145 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/2a1fc1cce69f4416aed60244320dddfb as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/2a1fc1cce69f4416aed60244320dddfb 2024-11-14T02:45:58,145 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/B of ac0d8410ac14c280aa06089ab1aa8285 into a582900796824bcfa8359198fb4b8029(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:58,145 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:58,145 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/B, priority=13, startTime=1731552357712; duration=0sec 2024-11-14T02:45:58,146 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:45:58,146 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:B 2024-11-14T02:45:58,146 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:45:58,147 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:45:58,147 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/C is initiating minor compaction (all files) 2024-11-14T02:45:58,147 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/C in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:58,147 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/148454cea4b04406a5c993bc157db3dd, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c8d6a40369c94f82b499a9106ce3a2bd, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/b2413d0796cd4d57993afcfc1d98305c] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=36.3 K 2024-11-14T02:45:58,148 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 148454cea4b04406a5c993bc157db3dd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731552353083 2024-11-14T02:45:58,148 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8d6a40369c94f82b499a9106ce3a2bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731552354268 2024-11-14T02:45:58,148 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2413d0796cd4d57993afcfc1d98305c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731552356463 2024-11-14T02:45:58,150 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/A of ac0d8410ac14c280aa06089ab1aa8285 into 2a1fc1cce69f4416aed60244320dddfb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:58,150 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:58,150 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/A, priority=13, startTime=1731552357712; duration=0sec 2024-11-14T02:45:58,150 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:58,150 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:A 2024-11-14T02:45:58,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/14fa66a85ce047e8ab912225a19f08c1 2024-11-14T02:45:58,153 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#C#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:45:58,153 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/7056f300ba5b4cdd9d0c7e24a75b128f is 50, key is test_row_0/C:col10/1731552356465/Put/seqid=0 2024-11-14T02:45:58,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742280_1456 (size=12983) 2024-11-14T02:45:58,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/4eb67314f99e4a3eb28278f5bc0e46f7 is 50, key is test_row_0/B:col10/1731552356584/Put/seqid=0 2024-11-14T02:45:58,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742281_1457 (size=12301) 2024-11-14T02:45:58,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:58,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:58,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:58,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:58,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:58,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:58,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552418405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552418405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:58,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:58,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552418415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:58,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:58,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:58,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:58,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:58,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T02:45:58,560 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/7056f300ba5b4cdd9d0c7e24a75b128f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/7056f300ba5b4cdd9d0c7e24a75b128f 2024-11-14T02:45:58,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/4eb67314f99e4a3eb28278f5bc0e46f7 2024-11-14T02:45:58,564 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/C of ac0d8410ac14c280aa06089ab1aa8285 into 7056f300ba5b4cdd9d0c7e24a75b128f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:45:58,564 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:58,564 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/C, priority=13, startTime=1731552357712; duration=0sec 2024-11-14T02:45:58,564 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:45:58,564 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:C 2024-11-14T02:45:58,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/46808b6773234fcfbee03decd4240878 is 50, key is test_row_0/C:col10/1731552356584/Put/seqid=0 2024-11-14T02:45:58,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742282_1458 (size=12301) 2024-11-14T02:45:58,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:58,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:58,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:58,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:58,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:58,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:58,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:58,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:58,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:58,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57076 deadline: 1731552418916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57148 deadline: 1731552418916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:58,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:45:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731552418925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:45:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 , the old value is region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:45:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=2 because the exception is null or not the one we care about 2024-11-14T02:45:58,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/46808b6773234fcfbee03decd4240878 2024-11-14T02:45:58,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/14fa66a85ce047e8ab912225a19f08c1 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/14fa66a85ce047e8ab912225a19f08c1 2024-11-14T02:45:58,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/14fa66a85ce047e8ab912225a19f08c1, entries=200, sequenceid=300, filesize=14.4 K 2024-11-14T02:45:58,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/4eb67314f99e4a3eb28278f5bc0e46f7 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/4eb67314f99e4a3eb28278f5bc0e46f7 2024-11-14T02:45:58,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/4eb67314f99e4a3eb28278f5bc0e46f7, entries=150, sequenceid=300, filesize=12.0 K 2024-11-14T02:45:58,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:58,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:58,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:58,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:58,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:58,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/46808b6773234fcfbee03decd4240878 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/46808b6773234fcfbee03decd4240878 2024-11-14T02:45:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/46808b6773234fcfbee03decd4240878, entries=150, sequenceid=300, filesize=12.0 K 2024-11-14T02:45:59,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for ac0d8410ac14c280aa06089ab1aa8285 in 1263ms, sequenceid=300, compaction requested=false 2024-11-14T02:45:59,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:45:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:45:59,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:45:59,097 INFO [Thread-1789 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1789. 2024-11-14T02:45:59,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:45:59,097 DEBUG [Thread-1789 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:59,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:59,097 DEBUG [Thread-1789 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:59,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:45:59,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:59,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:45:59,097 DEBUG [Thread-1789 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:59,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:45:59,097 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:59,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/b8de77913de343029adbfbd32280325a is 50, key is test_row_0/A:col10/1731552359095/Put/seqid=0 2024-11-14T02:45:59,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742283_1459 (size=12301) 2024-11-14T02:45:59,137 INFO [Thread-1793 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1793. 2024-11-14T02:45:59,137 DEBUG [Thread-1793 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:59,137 DEBUG [Thread-1793 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:59,137 DEBUG [Thread-1793 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:59,137 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:59,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:59,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:59,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:59,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:59,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:59,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:59,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:59,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:59,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:59,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:59,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:59,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:59,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/b8de77913de343029adbfbd32280325a 2024-11-14T02:45:59,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/ad5426043883467599b6c173675b4132 is 50, key is test_row_0/B:col10/1731552359095/Put/seqid=0 2024-11-14T02:45:59,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742284_1460 (size=12301) 2024-11-14T02:45:59,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:59,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:59,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:59,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:59,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:59,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:59,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,927 INFO [Thread-1795 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1795. 2024-11-14T02:45:59,927 DEBUG [Thread-1795 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:59,927 DEBUG [Thread-1795 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:59,927 DEBUG [Thread-1795 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:59,927 INFO [Thread-1787 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1787. 2024-11-14T02:45:59,927 DEBUG [Thread-1787 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:59,927 DEBUG [Thread-1787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:59,927 DEBUG [Thread-1787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:59,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:59,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:45:59,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:59,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:59,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:45:59,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:45:59,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:45:59,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/ad5426043883467599b6c173675b4132 2024-11-14T02:45:59,937 INFO [Thread-1791 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1791. 2024-11-14T02:45:59,937 DEBUG [Thread-1791 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:45:59,937 DEBUG [Thread-1791 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:59,937 DEBUG [Thread-1791 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:45:59,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:45:59,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/0f56d898246841a8aef15087559c0f09 is 50, key is test_row_0/C:col10/1731552359095/Put/seqid=0 2024-11-14T02:45:59,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742285_1461 (size=12301) 2024-11-14T02:46:00,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:46:00,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:46:00,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:46:00,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:46:00,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:00,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:00,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:46:00,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:46:00,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. as already flushing 2024-11-14T02:46:00,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:46:00,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:00,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:00,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/0f56d898246841a8aef15087559c0f09 2024-11-14T02:46:00,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/b8de77913de343029adbfbd32280325a as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/b8de77913de343029adbfbd32280325a 2024-11-14T02:46:00,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/b8de77913de343029adbfbd32280325a, entries=150, sequenceid=313, filesize=12.0 K 2024-11-14T02:46:00,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/ad5426043883467599b6c173675b4132 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/ad5426043883467599b6c173675b4132 2024-11-14T02:46:00,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/ad5426043883467599b6c173675b4132, entries=150, sequenceid=313, filesize=12.0 K 2024-11-14T02:46:00,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/0f56d898246841a8aef15087559c0f09 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/0f56d898246841a8aef15087559c0f09 2024-11-14T02:46:00,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/0f56d898246841a8aef15087559c0f09, entries=150, sequenceid=313, filesize=12.0 K 2024-11-14T02:46:00,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=26.84 KB/27480 for ac0d8410ac14c280aa06089ab1aa8285 in 1262ms, sequenceid=313, compaction requested=true 2024-11-14T02:46:00,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:46:00,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:46:00,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:00,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:46:00,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:00,359 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:00,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac0d8410ac14c280aa06089ab1aa8285:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:46:00,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:00,359 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:00,360 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:00,360 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:00,360 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/A is initiating minor compaction (all files) 2024-11-14T02:46:00,360 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/B is initiating minor compaction (all files) 2024-11-14T02:46:00,360 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/A in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:46:00,360 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/B in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:46:00,360 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/2a1fc1cce69f4416aed60244320dddfb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/14fa66a85ce047e8ab912225a19f08c1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/b8de77913de343029adbfbd32280325a] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=39.1 K 2024-11-14T02:46:00,360 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/a582900796824bcfa8359198fb4b8029, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/4eb67314f99e4a3eb28278f5bc0e46f7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/ad5426043883467599b6c173675b4132] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=36.7 K 2024-11-14T02:46:00,360 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a1fc1cce69f4416aed60244320dddfb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731552356463 2024-11-14T02:46:00,360 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting a582900796824bcfa8359198fb4b8029, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731552356463 2024-11-14T02:46:00,360 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14fa66a85ce047e8ab912225a19f08c1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731552356528 2024-11-14T02:46:00,360 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eb67314f99e4a3eb28278f5bc0e46f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731552356528 2024-11-14T02:46:00,361 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8de77913de343029adbfbd32280325a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731552357770 2024-11-14T02:46:00,361 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting ad5426043883467599b6c173675b4132, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731552357770 2024-11-14T02:46:00,368 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#B#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:00,369 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/75e954720b784dc1995c3e04df8b90fc is 50, key is test_row_0/B:col10/1731552359095/Put/seqid=0 2024-11-14T02:46:00,370 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#A#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:00,370 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/cfb1c0b7ffc14c82a327e65231fae6c8 is 50, key is test_row_0/A:col10/1731552359095/Put/seqid=0 2024-11-14T02:46:00,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742286_1462 (size=13085) 2024-11-14T02:46:00,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742287_1463 (size=13085) 2024-11-14T02:46:00,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T02:46:00,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:46:00,387 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing ac0d8410ac14c280aa06089ab1aa8285 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-14T02:46:00,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=A 2024-11-14T02:46:00,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:00,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=B 2024-11-14T02:46:00,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:00,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac0d8410ac14c280aa06089ab1aa8285, store=C 2024-11-14T02:46:00,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:00,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/8715d4e810624c9689171b894af74b32 is 50, key is test_row_1/A:col10/1731552359936/Put/seqid=0 2024-11-14T02:46:00,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742288_1464 (size=9857) 2024-11-14T02:46:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T02:46:00,777 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/75e954720b784dc1995c3e04df8b90fc as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/75e954720b784dc1995c3e04df8b90fc 2024-11-14T02:46:00,778 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/cfb1c0b7ffc14c82a327e65231fae6c8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/cfb1c0b7ffc14c82a327e65231fae6c8 2024-11-14T02:46:00,782 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/A of ac0d8410ac14c280aa06089ab1aa8285 into cfb1c0b7ffc14c82a327e65231fae6c8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:00,782 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/B of ac0d8410ac14c280aa06089ab1aa8285 into 75e954720b784dc1995c3e04df8b90fc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:00,782 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:46:00,782 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:46:00,782 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/A, priority=13, startTime=1731552360359; duration=0sec 2024-11-14T02:46:00,782 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/B, priority=13, startTime=1731552360359; duration=0sec 2024-11-14T02:46:00,782 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:00,782 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:00,782 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:A 2024-11-14T02:46:00,782 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:B 2024-11-14T02:46:00,782 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:00,783 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:00,783 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): ac0d8410ac14c280aa06089ab1aa8285/C is initiating minor compaction (all files) 2024-11-14T02:46:00,783 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac0d8410ac14c280aa06089ab1aa8285/C in TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:46:00,783 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/7056f300ba5b4cdd9d0c7e24a75b128f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/46808b6773234fcfbee03decd4240878, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/0f56d898246841a8aef15087559c0f09] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp, totalSize=36.7 K 2024-11-14T02:46:00,783 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7056f300ba5b4cdd9d0c7e24a75b128f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731552356463 2024-11-14T02:46:00,784 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46808b6773234fcfbee03decd4240878, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731552356528 2024-11-14T02:46:00,784 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f56d898246841a8aef15087559c0f09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731552357770 2024-11-14T02:46:00,791 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac0d8410ac14c280aa06089ab1aa8285#C#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:00,791 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/00dfcd952cbf4621a17cbb60678b6c59 is 50, key is test_row_0/C:col10/1731552359095/Put/seqid=0 2024-11-14T02:46:00,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742289_1465 (size=13085) 2024-11-14T02:46:00,794 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/8715d4e810624c9689171b894af74b32 2024-11-14T02:46:00,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/1a1991c87eb641e09cef2bfe768a68b3 is 50, key is test_row_1/B:col10/1731552359936/Put/seqid=0 2024-11-14T02:46:00,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742290_1466 (size=9857) 2024-11-14T02:46:01,199 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/00dfcd952cbf4621a17cbb60678b6c59 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/00dfcd952cbf4621a17cbb60678b6c59 2024-11-14T02:46:01,202 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/1a1991c87eb641e09cef2bfe768a68b3 2024-11-14T02:46:01,203 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac0d8410ac14c280aa06089ab1aa8285/C of ac0d8410ac14c280aa06089ab1aa8285 into 00dfcd952cbf4621a17cbb60678b6c59(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:01,203 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:46:01,203 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285., storeName=ac0d8410ac14c280aa06089ab1aa8285/C, priority=13, startTime=1731552360359; duration=0sec 2024-11-14T02:46:01,203 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:01,203 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac0d8410ac14c280aa06089ab1aa8285:C 2024-11-14T02:46:01,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/3ae9daddf0784f06a3e8a247feac2354 is 50, key is test_row_1/C:col10/1731552359936/Put/seqid=0 2024-11-14T02:46:01,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742291_1467 (size=9857) 2024-11-14T02:46:01,613 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/3ae9daddf0784f06a3e8a247feac2354 2024-11-14T02:46:01,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/A/8715d4e810624c9689171b894af74b32 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/8715d4e810624c9689171b894af74b32 2024-11-14T02:46:01,623 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/8715d4e810624c9689171b894af74b32, entries=100, sequenceid=320, filesize=9.6 K 2024-11-14T02:46:01,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/B/1a1991c87eb641e09cef2bfe768a68b3 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/1a1991c87eb641e09cef2bfe768a68b3 2024-11-14T02:46:01,629 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/1a1991c87eb641e09cef2bfe768a68b3, entries=100, sequenceid=320, filesize=9.6 K 2024-11-14T02:46:01,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/.tmp/C/3ae9daddf0784f06a3e8a247feac2354 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3ae9daddf0784f06a3e8a247feac2354 2024-11-14T02:46:01,635 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3ae9daddf0784f06a3e8a247feac2354, entries=100, sequenceid=320, filesize=9.6 K 2024-11-14T02:46:01,636 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ac0d8410ac14c280aa06089ab1aa8285 in 1248ms, sequenceid=320, compaction requested=false 2024-11-14T02:46:01,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for ac0d8410ac14c280aa06089ab1aa8285: 2024-11-14T02:46:01,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:46:01,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-14T02:46:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-14T02:46:01,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-14T02:46:01,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.2370 sec 2024-11-14T02:46:01,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 5.2450 sec 2024-11-14T02:46:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T02:46:04,576 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1275 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3823 rows 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1265 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3794 rows 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1270 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3805 rows 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1271 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3813 rows 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1272 2024-11-14T02:46:04,576 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3816 rows 2024-11-14T02:46:04,576 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:46:04,576 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:04,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:04,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:04,577 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:04,577 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T02:46:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:04,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T02:46:04,580 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552364580"}]},"ts":"1731552364580"} 2024-11-14T02:46:04,582 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T02:46:04,582 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T02:46:04,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:46:04,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac0d8410ac14c280aa06089ab1aa8285, UNASSIGN}] 2024-11-14T02:46:04,584 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac0d8410ac14c280aa06089ab1aa8285, UNASSIGN 2024-11-14T02:46:04,585 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=ac0d8410ac14c280aa06089ab1aa8285, regionState=CLOSING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:04,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac0d8410ac14c280aa06089ab1aa8285, UNASSIGN because future has completed 2024-11-14T02:46:04,586 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:46:04,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:46:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T02:46:04,740 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(122): Close ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:46:04,740 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:46:04,740 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1722): Closing ac0d8410ac14c280aa06089ab1aa8285, disabling compactions & flushes 2024-11-14T02:46:04,740 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:46:04,740 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:46:04,740 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. after waiting 0 ms 2024-11-14T02:46:04,741 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:46:04,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/37d967b862454008ac2b41cea52b9f91, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/a501c50c2fcc41fea55dd86d6b61fee0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/678ee9ed963d4f75a098b8b622cd8847, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/afc258ed450c4094bf2dfd051e5c35f9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/3653f72cd8cf4df1ba0ba7e06ab235b8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/9ad92f7286b84e839fa8ebdff4f56688, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/7f81b79493b44ca4b7a1f09140981aad, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/3b8899cd55f3463289b62c08438719f1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/1c1b791c906d4a4c947449a16ace4dfb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/5a4f4fb0278042a68952d72d848071a2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/d9ce3193667a40cc875721eb6259c92a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/abc89cf6b6bc4f6c9b75d41a70b5f726, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/0c8854924b0149ce9c7e4a623d2d0b53, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/7023025de17f47818e2eb157b49e06e9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/ac0bcc62675a40aa8b176595a4e8143a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/da6715c449b34f4d899df575eacde034, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/9b1b5a1cf62e4c528c8eec4e5c1c3509, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/22dd0e2572e24af2b9e4ee190c39a79a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/fa43d266e0ac4b47b6657b9236a47824, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/58cd6d872aca424d91ca0f3cfee5b8e7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/2a1fc1cce69f4416aed60244320dddfb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/14fa66a85ce047e8ab912225a19f08c1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/b8de77913de343029adbfbd32280325a] to archive 2024-11-14T02:46:04,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:46:04,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/37d967b862454008ac2b41cea52b9f91 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/37d967b862454008ac2b41cea52b9f91 2024-11-14T02:46:04,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/a501c50c2fcc41fea55dd86d6b61fee0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/a501c50c2fcc41fea55dd86d6b61fee0 2024-11-14T02:46:04,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/678ee9ed963d4f75a098b8b622cd8847 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/678ee9ed963d4f75a098b8b622cd8847 2024-11-14T02:46:04,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/afc258ed450c4094bf2dfd051e5c35f9 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/afc258ed450c4094bf2dfd051e5c35f9 2024-11-14T02:46:04,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/3653f72cd8cf4df1ba0ba7e06ab235b8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/3653f72cd8cf4df1ba0ba7e06ab235b8 2024-11-14T02:46:04,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/9ad92f7286b84e839fa8ebdff4f56688 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/9ad92f7286b84e839fa8ebdff4f56688 2024-11-14T02:46:04,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/7f81b79493b44ca4b7a1f09140981aad to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/7f81b79493b44ca4b7a1f09140981aad 2024-11-14T02:46:04,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/3b8899cd55f3463289b62c08438719f1 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/3b8899cd55f3463289b62c08438719f1 2024-11-14T02:46:04,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/1c1b791c906d4a4c947449a16ace4dfb to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/1c1b791c906d4a4c947449a16ace4dfb 2024-11-14T02:46:04,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/5a4f4fb0278042a68952d72d848071a2 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/5a4f4fb0278042a68952d72d848071a2 2024-11-14T02:46:04,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/d9ce3193667a40cc875721eb6259c92a to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/d9ce3193667a40cc875721eb6259c92a 2024-11-14T02:46:04,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/abc89cf6b6bc4f6c9b75d41a70b5f726 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/abc89cf6b6bc4f6c9b75d41a70b5f726 2024-11-14T02:46:04,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/0c8854924b0149ce9c7e4a623d2d0b53 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/0c8854924b0149ce9c7e4a623d2d0b53 2024-11-14T02:46:04,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/7023025de17f47818e2eb157b49e06e9 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/7023025de17f47818e2eb157b49e06e9 2024-11-14T02:46:04,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/ac0bcc62675a40aa8b176595a4e8143a to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/ac0bcc62675a40aa8b176595a4e8143a 2024-11-14T02:46:04,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/da6715c449b34f4d899df575eacde034 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/da6715c449b34f4d899df575eacde034 2024-11-14T02:46:04,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/9b1b5a1cf62e4c528c8eec4e5c1c3509 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/9b1b5a1cf62e4c528c8eec4e5c1c3509 2024-11-14T02:46:04,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/22dd0e2572e24af2b9e4ee190c39a79a to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/22dd0e2572e24af2b9e4ee190c39a79a 2024-11-14T02:46:04,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/fa43d266e0ac4b47b6657b9236a47824 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/fa43d266e0ac4b47b6657b9236a47824 2024-11-14T02:46:04,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/58cd6d872aca424d91ca0f3cfee5b8e7 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/58cd6d872aca424d91ca0f3cfee5b8e7 2024-11-14T02:46:04,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/2a1fc1cce69f4416aed60244320dddfb to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/2a1fc1cce69f4416aed60244320dddfb 2024-11-14T02:46:04,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/14fa66a85ce047e8ab912225a19f08c1 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/14fa66a85ce047e8ab912225a19f08c1 2024-11-14T02:46:04,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/b8de77913de343029adbfbd32280325a to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/b8de77913de343029adbfbd32280325a 2024-11-14T02:46:04,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/e2f18a87f3634aa3833c09948d31d406, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/6db8733203ac42909751d5dfd3b33df0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/e49ca53dbdec4b47af2563d698f45737, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/ab8dad905e154f11bb66d58acf580d51, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/c11f4763d0294243a2145a39e58e087c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/493b39127fa7449e8d510e70bfa59348, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/49b192b607694c8c9990ce560fb0c6bd, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/61abf24192a7419a989d9abc79b05416, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/5e882e18e1d040acaa315e49a44d35cc, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/45d0f06aeea349a99de89a500b191909, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/66992749e2a14221b4edc7d0b6e44210, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/cae6ec144cf1432dad2cf8d5985d400b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/835f4d95efea4e96b85ed2d66c74a7a5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/8dbde70824e54ea8aeb1e8a8ea283b5c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/3089f28c2222434788d278fa7dbcc430, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/50ff6ba533a241779a452870da8dc0bf, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/cc7da23b01a84899a8ce196bc623d0cb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/66cc5cca38774162b7d5111ba91cfcab, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/6cca875bfcd54c6fbd37a5078df55e8c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/a582900796824bcfa8359198fb4b8029, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/1c9fba41601f4a61841e712ef124c53f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/4eb67314f99e4a3eb28278f5bc0e46f7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/ad5426043883467599b6c173675b4132] to archive 2024-11-14T02:46:04,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:46:04,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/e2f18a87f3634aa3833c09948d31d406 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/e2f18a87f3634aa3833c09948d31d406 2024-11-14T02:46:04,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/6db8733203ac42909751d5dfd3b33df0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/6db8733203ac42909751d5dfd3b33df0 2024-11-14T02:46:04,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/e49ca53dbdec4b47af2563d698f45737 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/e49ca53dbdec4b47af2563d698f45737 2024-11-14T02:46:04,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/ab8dad905e154f11bb66d58acf580d51 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/ab8dad905e154f11bb66d58acf580d51 2024-11-14T02:46:04,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/c11f4763d0294243a2145a39e58e087c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/c11f4763d0294243a2145a39e58e087c 2024-11-14T02:46:04,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/493b39127fa7449e8d510e70bfa59348 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/493b39127fa7449e8d510e70bfa59348 2024-11-14T02:46:04,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/49b192b607694c8c9990ce560fb0c6bd to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/49b192b607694c8c9990ce560fb0c6bd 2024-11-14T02:46:04,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/61abf24192a7419a989d9abc79b05416 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/61abf24192a7419a989d9abc79b05416 2024-11-14T02:46:04,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/5e882e18e1d040acaa315e49a44d35cc to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/5e882e18e1d040acaa315e49a44d35cc 2024-11-14T02:46:04,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/45d0f06aeea349a99de89a500b191909 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/45d0f06aeea349a99de89a500b191909 2024-11-14T02:46:04,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/66992749e2a14221b4edc7d0b6e44210 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/66992749e2a14221b4edc7d0b6e44210 2024-11-14T02:46:04,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/cae6ec144cf1432dad2cf8d5985d400b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/cae6ec144cf1432dad2cf8d5985d400b 2024-11-14T02:46:04,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/835f4d95efea4e96b85ed2d66c74a7a5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/835f4d95efea4e96b85ed2d66c74a7a5 2024-11-14T02:46:04,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/8dbde70824e54ea8aeb1e8a8ea283b5c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/8dbde70824e54ea8aeb1e8a8ea283b5c 2024-11-14T02:46:04,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/3089f28c2222434788d278fa7dbcc430 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/3089f28c2222434788d278fa7dbcc430 2024-11-14T02:46:04,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/50ff6ba533a241779a452870da8dc0bf to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/50ff6ba533a241779a452870da8dc0bf 2024-11-14T02:46:04,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/cc7da23b01a84899a8ce196bc623d0cb to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/cc7da23b01a84899a8ce196bc623d0cb 2024-11-14T02:46:04,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/66cc5cca38774162b7d5111ba91cfcab to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/66cc5cca38774162b7d5111ba91cfcab 2024-11-14T02:46:04,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/6cca875bfcd54c6fbd37a5078df55e8c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/6cca875bfcd54c6fbd37a5078df55e8c 2024-11-14T02:46:04,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/a582900796824bcfa8359198fb4b8029 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/a582900796824bcfa8359198fb4b8029 2024-11-14T02:46:04,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/1c9fba41601f4a61841e712ef124c53f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/1c9fba41601f4a61841e712ef124c53f 2024-11-14T02:46:04,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/4eb67314f99e4a3eb28278f5bc0e46f7 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/4eb67314f99e4a3eb28278f5bc0e46f7 2024-11-14T02:46:04,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/ad5426043883467599b6c173675b4132 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/ad5426043883467599b6c173675b4132 2024-11-14T02:46:04,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/d4946b7a71e745239d06134c423653f5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/a506a593276e47cb9bcf03e7c90206cc, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c46b15a11ffc4321a4fbe61c3352e758, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/db82982188514c55a6bd1d141b5c4081, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c50a4d93197a446f93326e709cf2f1ac, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/ad12855e92174c249e1242f6b32a24d3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3766fc66c38c415292099cd188c148d0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/cf007a24743d472cb43c87acaf78a962, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/63f2c48fe53d46448a2c14c26009e5cf, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/0db76944ca5f4bcda45611cf24cadde2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/67f00eaae26b46a09cae66eff56d7ef1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/56faed4dbdb8446990d05ca1cf4b9409, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/8ecdb9ac2fbc4b8ebffb0dd78295af0f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3f6887f92af242e29e8512e85981de41, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/cc3584aca4ff4bdc93cf9983f7af41e8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/fa4ac9d354d34f64b71d8e3769f6cb86, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/148454cea4b04406a5c993bc157db3dd, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/ec89236cbcdb4a01b1aaff0218cf9c71, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c8d6a40369c94f82b499a9106ce3a2bd, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/7056f300ba5b4cdd9d0c7e24a75b128f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/b2413d0796cd4d57993afcfc1d98305c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/46808b6773234fcfbee03decd4240878, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/0f56d898246841a8aef15087559c0f09] to archive 2024-11-14T02:46:04,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:46:04,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/d4946b7a71e745239d06134c423653f5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/d4946b7a71e745239d06134c423653f5 2024-11-14T02:46:04,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/a506a593276e47cb9bcf03e7c90206cc to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/a506a593276e47cb9bcf03e7c90206cc 2024-11-14T02:46:04,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c46b15a11ffc4321a4fbe61c3352e758 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c46b15a11ffc4321a4fbe61c3352e758 2024-11-14T02:46:04,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/db82982188514c55a6bd1d141b5c4081 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/db82982188514c55a6bd1d141b5c4081 2024-11-14T02:46:04,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c50a4d93197a446f93326e709cf2f1ac to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c50a4d93197a446f93326e709cf2f1ac 2024-11-14T02:46:04,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/ad12855e92174c249e1242f6b32a24d3 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/ad12855e92174c249e1242f6b32a24d3 2024-11-14T02:46:04,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3766fc66c38c415292099cd188c148d0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3766fc66c38c415292099cd188c148d0 2024-11-14T02:46:04,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/cf007a24743d472cb43c87acaf78a962 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/cf007a24743d472cb43c87acaf78a962 2024-11-14T02:46:04,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/63f2c48fe53d46448a2c14c26009e5cf to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/63f2c48fe53d46448a2c14c26009e5cf 2024-11-14T02:46:04,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/0db76944ca5f4bcda45611cf24cadde2 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/0db76944ca5f4bcda45611cf24cadde2 2024-11-14T02:46:04,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/67f00eaae26b46a09cae66eff56d7ef1 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/67f00eaae26b46a09cae66eff56d7ef1 2024-11-14T02:46:04,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/56faed4dbdb8446990d05ca1cf4b9409 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/56faed4dbdb8446990d05ca1cf4b9409 2024-11-14T02:46:04,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/8ecdb9ac2fbc4b8ebffb0dd78295af0f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/8ecdb9ac2fbc4b8ebffb0dd78295af0f 2024-11-14T02:46:04,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3f6887f92af242e29e8512e85981de41 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3f6887f92af242e29e8512e85981de41 2024-11-14T02:46:04,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/cc3584aca4ff4bdc93cf9983f7af41e8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/cc3584aca4ff4bdc93cf9983f7af41e8 2024-11-14T02:46:04,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/fa4ac9d354d34f64b71d8e3769f6cb86 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/fa4ac9d354d34f64b71d8e3769f6cb86 2024-11-14T02:46:04,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/148454cea4b04406a5c993bc157db3dd to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/148454cea4b04406a5c993bc157db3dd 2024-11-14T02:46:04,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/ec89236cbcdb4a01b1aaff0218cf9c71 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/ec89236cbcdb4a01b1aaff0218cf9c71 2024-11-14T02:46:04,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c8d6a40369c94f82b499a9106ce3a2bd to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/c8d6a40369c94f82b499a9106ce3a2bd 2024-11-14T02:46:04,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/7056f300ba5b4cdd9d0c7e24a75b128f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/7056f300ba5b4cdd9d0c7e24a75b128f 2024-11-14T02:46:04,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/b2413d0796cd4d57993afcfc1d98305c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/b2413d0796cd4d57993afcfc1d98305c 2024-11-14T02:46:04,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/46808b6773234fcfbee03decd4240878 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/46808b6773234fcfbee03decd4240878 2024-11-14T02:46:04,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/0f56d898246841a8aef15087559c0f09 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/0f56d898246841a8aef15087559c0f09 2024-11-14T02:46:04,820 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/recovered.edits/326.seqid, newMaxSeqId=326, maxSeqId=1 2024-11-14T02:46:04,820 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285. 2024-11-14T02:46:04,820 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1676): Region close journal for ac0d8410ac14c280aa06089ab1aa8285: Waiting for close lock at 1731552364740Running coprocessor pre-close hooks at 1731552364740Disabling compacts and flushes for region at 1731552364740Disabling writes for close at 1731552364741 (+1 ms)Writing region close event to WAL at 1731552364817 (+76 ms)Running coprocessor post-close hooks at 1731552364820 (+3 ms)Closed at 1731552364820 2024-11-14T02:46:04,821 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(157): Closed ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:46:04,822 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=ac0d8410ac14c280aa06089ab1aa8285, regionState=CLOSED 2024-11-14T02:46:04,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:46:04,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-14T02:46:04,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseRegionProcedure ac0d8410ac14c280aa06089ab1aa8285, server=e5dc8b5cf5e9,46387,1731552206828 in 236 msec 2024-11-14T02:46:04,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-11-14T02:46:04,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac0d8410ac14c280aa06089ab1aa8285, UNASSIGN in 241 msec 2024-11-14T02:46:04,827 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-14T02:46:04,827 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 244 msec 2024-11-14T02:46:04,828 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552364828"}]},"ts":"1731552364828"} 2024-11-14T02:46:04,829 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T02:46:04,829 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T02:46:04,831 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 253 msec 2024-11-14T02:46:04,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T02:46:04,895 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:04,896 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T02:46:04,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:04,897 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T02:46:04,898 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=133, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:04,901 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:46:04,902 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/recovered.edits] 2024-11-14T02:46:04,905 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/8715d4e810624c9689171b894af74b32 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/8715d4e810624c9689171b894af74b32 2024-11-14T02:46:04,906 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/cfb1c0b7ffc14c82a327e65231fae6c8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/A/cfb1c0b7ffc14c82a327e65231fae6c8 2024-11-14T02:46:04,908 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/1a1991c87eb641e09cef2bfe768a68b3 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/1a1991c87eb641e09cef2bfe768a68b3 2024-11-14T02:46:04,908 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/75e954720b784dc1995c3e04df8b90fc to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/B/75e954720b784dc1995c3e04df8b90fc 2024-11-14T02:46:04,910 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/00dfcd952cbf4621a17cbb60678b6c59 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/00dfcd952cbf4621a17cbb60678b6c59 2024-11-14T02:46:04,910 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3ae9daddf0784f06a3e8a247feac2354 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/C/3ae9daddf0784f06a3e8a247feac2354 2024-11-14T02:46:04,912 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/recovered.edits/326.seqid to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285/recovered.edits/326.seqid 2024-11-14T02:46:04,912 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/ac0d8410ac14c280aa06089ab1aa8285 2024-11-14T02:46:04,912 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T02:46:04,914 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=133, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:04,916 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T02:46:04,917 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T02:46:04,918 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=133, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:04,918 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T02:46:04,918 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731552364918"}]},"ts":"9223372036854775807"} 2024-11-14T02:46:04,920 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T02:46:04,920 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ac0d8410ac14c280aa06089ab1aa8285, NAME => 'TestAcidGuarantees,,1731552335641.ac0d8410ac14c280aa06089ab1aa8285.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T02:46:04,920 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T02:46:04,920 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731552364920"}]},"ts":"9223372036854775807"} 2024-11-14T02:46:04,921 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T02:46:04,922 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=133, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:04,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 26 msec 2024-11-14T02:46:05,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T02:46:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T02:46:05,006 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:05,014 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=254 (was 251) - Thread LEAK? -, OpenFileDescriptor=463 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=323 (was 348), ProcessCount=11 (was 11), AvailableMemoryMB=9688 (was 9805) 2024-11-14T02:46:05,022 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=254, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=323, ProcessCount=11, AvailableMemoryMB=9687 2024-11-14T02:46:05,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:46:05,023 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:46:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:05,025 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T02:46:05,025 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:05,025 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 134 2024-11-14T02:46:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T02:46:05,026 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T02:46:05,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742292_1468 (size=1061) 2024-11-14T02:46:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T02:46:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T02:46:05,432 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1 2024-11-14T02:46:05,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742293_1469 (size=53) 2024-11-14T02:46:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T02:46:05,837 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:05,837 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 672d111a762fbc594a3a447bdac7fae8, disabling compactions & flushes 2024-11-14T02:46:05,837 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:05,837 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:05,837 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. after waiting 0 ms 2024-11-14T02:46:05,837 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:05,837 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:05,837 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 672d111a762fbc594a3a447bdac7fae8: Waiting for close lock at 1731552365837Disabling compacts and flushes for region at 1731552365837Disabling writes for close at 1731552365837Writing region close event to WAL at 1731552365837Closed at 1731552365837 2024-11-14T02:46:05,839 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T02:46:05,839 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731552365839"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731552365839"}]},"ts":"1731552365839"} 2024-11-14T02:46:05,841 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T02:46:05,842 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T02:46:05,842 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552365842"}]},"ts":"1731552365842"} 2024-11-14T02:46:05,843 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T02:46:05,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=672d111a762fbc594a3a447bdac7fae8, ASSIGN}] 2024-11-14T02:46:05,845 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=672d111a762fbc594a3a447bdac7fae8, ASSIGN 2024-11-14T02:46:05,846 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=672d111a762fbc594a3a447bdac7fae8, ASSIGN; state=OFFLINE, location=e5dc8b5cf5e9,46387,1731552206828; forceNewPlan=false, retain=false 2024-11-14T02:46:05,997 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=672d111a762fbc594a3a447bdac7fae8, regionState=OPENING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:06,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=672d111a762fbc594a3a447bdac7fae8, ASSIGN because future has completed 2024-11-14T02:46:06,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:46:06,155 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:06,155 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7752): Opening region: {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:46:06,156 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:06,156 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:06,156 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7794): checking encryption for 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:06,156 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7797): checking classloading for 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:06,157 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:06,159 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:46:06,159 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 672d111a762fbc594a3a447bdac7fae8 columnFamilyName A 2024-11-14T02:46:06,159 DEBUG [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:06,160 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.HStore(327): Store=672d111a762fbc594a3a447bdac7fae8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:06,160 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:06,161 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:46:06,161 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 672d111a762fbc594a3a447bdac7fae8 columnFamilyName B 2024-11-14T02:46:06,161 DEBUG [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:06,162 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.HStore(327): Store=672d111a762fbc594a3a447bdac7fae8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:06,162 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:06,163 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:46:06,163 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 672d111a762fbc594a3a447bdac7fae8 columnFamilyName C 2024-11-14T02:46:06,163 DEBUG [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:06,164 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.HStore(327): Store=672d111a762fbc594a3a447bdac7fae8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:06,164 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:06,164 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1038): replaying wal for 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:06,165 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:06,165 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T02:46:06,166 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1048): stopping wal replay for 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:06,166 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1060): Cleaning up temporary data for 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:06,166 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:46:06,167 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1093): writing seq id for 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:06,170 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:46:06,170 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1114): Opened 672d111a762fbc594a3a447bdac7fae8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63547496, jitterRate=-0.05306851863861084}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:46:06,170 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:06,171 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1006): Region open journal for 672d111a762fbc594a3a447bdac7fae8: Running coprocessor pre-open hook at 1731552366156Writing region info on filesystem at 1731552366156Initializing all the Stores at 1731552366157 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552366157Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552366157Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552366157Cleaning up temporary data from old regions at 1731552366166 (+9 ms)Running coprocessor post-open hooks at 1731552366170 (+4 ms)Region opened successfully at 1731552366171 (+1 ms) 2024-11-14T02:46:06,172 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., pid=136, masterSystemTime=1731552366152 2024-11-14T02:46:06,174 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:06,174 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:06,175 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=672d111a762fbc594a3a447bdac7fae8, regionState=OPEN, openSeqNum=2, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:06,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:46:06,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-14T02:46:06,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure 672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 in 179 msec 2024-11-14T02:46:06,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-14T02:46:06,186 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=672d111a762fbc594a3a447bdac7fae8, ASSIGN in 339 msec 2024-11-14T02:46:06,186 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T02:46:06,187 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552366187"}]},"ts":"1731552366187"} 2024-11-14T02:46:06,188 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T02:46:06,189 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T02:46:06,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-11-14T02:46:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T02:46:07,175 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:07,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22faf5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:07,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:46:07,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:46:07,178 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:46:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:46:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:46:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6085cd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:46:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:46:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:07,180 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:46:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b05621c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:07,181 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:46:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:46:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:07,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:07,185 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:46:07,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:46:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54ba6dc5 2024-11-14T02:46:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:46:07,187 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:46:07,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T02:46:07,189 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:46:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:07,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742294_1470 (size=1097) 2024-11-14T02:46:07,604 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T02:46:07,604 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T02:46:07,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:46:07,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=672d111a762fbc594a3a447bdac7fae8, REOPEN/MOVE}] 2024-11-14T02:46:07,613 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=672d111a762fbc594a3a447bdac7fae8, REOPEN/MOVE 2024-11-14T02:46:07,614 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=672d111a762fbc594a3a447bdac7fae8, regionState=CLOSING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:07,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=672d111a762fbc594a3a447bdac7fae8, REOPEN/MOVE because future has completed 2024-11-14T02:46:07,615 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:46:07,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:46:07,767 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:07,767 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:46:07,767 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 672d111a762fbc594a3a447bdac7fae8, disabling compactions & flushes 2024-11-14T02:46:07,767 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:07,767 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:07,767 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. after waiting 0 ms 2024-11-14T02:46:07,767 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:07,771 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T02:46:07,771 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:07,771 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 672d111a762fbc594a3a447bdac7fae8: Waiting for close lock at 1731552367767Running coprocessor pre-close hooks at 1731552367767Disabling compacts and flushes for region at 1731552367767Disabling writes for close at 1731552367767Writing region close event to WAL at 1731552367768 (+1 ms)Running coprocessor post-close hooks at 1731552367771 (+3 ms)Closed at 1731552367771 2024-11-14T02:46:07,771 WARN [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionServer(3299): Not adding moved region record: 672d111a762fbc594a3a447bdac7fae8 to self. 2024-11-14T02:46:07,773 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:07,774 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=672d111a762fbc594a3a447bdac7fae8, regionState=CLOSED 2024-11-14T02:46:07,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:46:07,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-14T02:46:07,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 in 161 msec 2024-11-14T02:46:07,779 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=672d111a762fbc594a3a447bdac7fae8, REOPEN/MOVE; state=CLOSED, location=e5dc8b5cf5e9,46387,1731552206828; forceNewPlan=false, retain=true 2024-11-14T02:46:07,929 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=672d111a762fbc594a3a447bdac7fae8, regionState=OPENING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:07,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=672d111a762fbc594a3a447bdac7fae8, REOPEN/MOVE because future has completed 2024-11-14T02:46:07,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:46:08,085 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:08,085 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7752): Opening region: {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:46:08,085 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,085 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:08,085 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7794): checking encryption for 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,085 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7797): checking classloading for 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,086 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,087 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:46:08,087 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 672d111a762fbc594a3a447bdac7fae8 columnFamilyName A 2024-11-14T02:46:08,088 DEBUG [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:08,088 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.HStore(327): Store=672d111a762fbc594a3a447bdac7fae8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:08,089 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,089 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:46:08,089 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 672d111a762fbc594a3a447bdac7fae8 columnFamilyName B 2024-11-14T02:46:08,089 DEBUG [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:08,090 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.HStore(327): Store=672d111a762fbc594a3a447bdac7fae8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:08,090 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,090 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T02:46:08,090 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 672d111a762fbc594a3a447bdac7fae8 columnFamilyName C 2024-11-14T02:46:08,090 DEBUG [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:08,091 INFO [StoreOpener-672d111a762fbc594a3a447bdac7fae8-1 {}] regionserver.HStore(327): Store=672d111a762fbc594a3a447bdac7fae8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:08,091 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:08,091 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1038): replaying wal for 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,091 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,092 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,093 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1048): stopping wal replay for 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,093 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1060): Cleaning up temporary data for 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,093 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T02:46:08,094 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): writing seq id for 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,095 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1114): Opened 672d111a762fbc594a3a447bdac7fae8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73409199, jitterRate=0.09388230741024017}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T02:46:08,095 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,096 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1006): Region open journal for 672d111a762fbc594a3a447bdac7fae8: Running coprocessor pre-open hook at 1731552368085Writing region info on filesystem at 1731552368085Initializing all the Stores at 1731552368086 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552368086Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552368086Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552368086Cleaning up temporary data from old regions at 1731552368093 (+7 ms)Running coprocessor post-open hooks at 1731552368095 (+2 ms)Region opened successfully at 1731552368096 (+1 ms) 2024-11-14T02:46:08,097 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., pid=141, masterSystemTime=1731552368082 2024-11-14T02:46:08,100 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:08,100 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:08,101 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=672d111a762fbc594a3a447bdac7fae8, regionState=OPEN, openSeqNum=5, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,102 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:46:08,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=139 2024-11-14T02:46:08,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 in 173 msec 2024-11-14T02:46:08,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-14T02:46:08,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=672d111a762fbc594a3a447bdac7fae8, REOPEN/MOVE in 493 msec 2024-11-14T02:46:08,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-14T02:46:08,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 500 msec 2024-11-14T02:46:08,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 922 msec 2024-11-14T02:46:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T02:46:08,114 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:08,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,115 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:46:08,115 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:46:08,116 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:46:08,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:46:08,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:46:08,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd13bb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:46:08,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:46:08,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:08,118 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:46:08,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68a90f8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,119 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:46:08,119 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:46:08,120 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:46:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:46:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:46:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6b14a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:46:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:46:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:08,121 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:46:08,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5021889f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,123 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:46:08,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:46:08,124 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:46:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:46:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:46:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c73f23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:46:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:46:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:08,125 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:46:08,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eecab4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,127 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:46:08,127 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:46:08,127 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:46:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:46:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:46:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@515fd540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:46:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:46:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:08,129 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:46:08,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aeb7db1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,130 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:46:08,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:46:08,131 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:46:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:46:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:46:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@653af5b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:46:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:46:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:08,133 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:46:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@136ed2be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,134 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:46:08,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:46:08,135 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:46:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:46:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:46:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e0ba23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:46:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:46:08,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:08,136 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:46:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66721380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,138 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:46:08,138 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:46:08,139 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:46:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:46:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:46:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f7624e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:46:08,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:46:08,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:08,141 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:46:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23aef601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,142 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,142 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:46:08,142 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:46:08,143 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:46:08,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:46:08,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:46:08,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b9c9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:46:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:46:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:08,145 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:46:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38dc011c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,146 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:46:08,146 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:46:08,147 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:46:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:46:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:46:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7505ce6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:46:08,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:46:08,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:08,149 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:46:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f309e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2517816d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,150 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,38717,-1 for getting cluster id 2024-11-14T02:46:08,150 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:46:08,151 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6372a0b-df7a-4882-85af-70267aaacec2' 2024-11-14T02:46:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:46:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6372a0b-df7a-4882-85af-70267aaacec2" 2024-11-14T02:46:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38af19f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,38717,-1] 2024-11-14T02:46:08,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:46:08,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:08,153 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:46:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f95e6c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:08,154 DEBUG [Thread-2145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:46:08,154 DEBUG [Thread-2143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:46:08,154 DEBUG [Thread-2136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:46:08,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:46:08,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:46:08,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:46:08,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:08,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:08,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:08,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:08,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:08,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:46:08,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:46:08,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:08,160 DEBUG [Thread-2134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:46:08,160 DEBUG [Thread-2151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:46:08,161 DEBUG [Thread-2149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:46:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:46:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:46:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:46:08,162 DEBUG [Thread-2138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:46:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:08,163 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:08,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:46:08,163 DEBUG [Thread-2132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:46:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:46:08,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:08,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:46:08,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:46:08,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:08,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:08,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:46:08,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:08,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:46:08,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:46:08,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:46:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T02:46:08,177 DEBUG [Thread-2140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:46:08,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:08,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:08,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:08,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:08,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:08,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:08,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:46:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:08,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:08,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:46:08,202 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:08,202 DEBUG [Thread-2147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:46:08,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=-1] 2024-11-14T02:46:08,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:08,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5] 2024-11-14T02:46:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-14T02:46:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T02:46:08,209 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:08,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552368196, max timestamp=1731552368205Num uniques 859; 2024-11-14T02:46:08,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:46:08,210 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:08,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:08,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552368196, max timestamp=1731552368205Num uniques 859; 2024-11-14T02:46:08,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552368196, max timestamp=1731552368205Num uniques 859; 2024-11-14T02:46:08,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:46:08,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:46:08,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141a0699942d704ed8b16d11840e183367_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552368173/Put/seqid=0 2024-11-14T02:46:08,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552428234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552428234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552428236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552428238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552428241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742295_1471 (size=12154) 2024-11-14T02:46:08,256 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:08,260 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141a0699942d704ed8b16d11840e183367_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141a0699942d704ed8b16d11840e183367_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,261 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/df554a3240454e0593af964f47297fe3, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:08,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/df554a3240454e0593af964f47297fe3 is 175, key is test_row_0/A:col10/1731552368173/Put/seqid=0 2024-11-14T02:46:08,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742296_1472 (size=30955) 2024-11-14T02:46:08,274 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/df554a3240454e0593af964f47297fe3 2024-11-14T02:46:08,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/c24927d1409a40748bf92bfe82685c97 is 50, key is test_row_0/B:col10/1731552368173/Put/seqid=0 2024-11-14T02:46:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T02:46:08,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742297_1473 (size=12001) 2024-11-14T02:46:08,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/c24927d1409a40748bf92bfe82685c97 2024-11-14T02:46:08,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552428346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552428346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/fb4039ebdc87407b951a8daf2ced6df1 is 50, key is test_row_0/C:col10/1731552368173/Put/seqid=0 2024-11-14T02:46:08,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552428355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552428355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552428355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T02:46:08,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:08,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:08,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:08,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:08,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:08,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742298_1474 (size=12001) 2024-11-14T02:46:08,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/fb4039ebdc87407b951a8daf2ced6df1 2024-11-14T02:46:08,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/df554a3240454e0593af964f47297fe3 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/df554a3240454e0593af964f47297fe3 2024-11-14T02:46:08,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/df554a3240454e0593af964f47297fe3, entries=150, sequenceid=14, filesize=30.2 K 2024-11-14T02:46:08,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/c24927d1409a40748bf92bfe82685c97 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/c24927d1409a40748bf92bfe82685c97 2024-11-14T02:46:08,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/c24927d1409a40748bf92bfe82685c97, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T02:46:08,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/fb4039ebdc87407b951a8daf2ced6df1 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/fb4039ebdc87407b951a8daf2ced6df1 2024-11-14T02:46:08,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/fb4039ebdc87407b951a8daf2ced6df1, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T02:46:08,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 672d111a762fbc594a3a447bdac7fae8 in 256ms, sequenceid=14, compaction requested=false 2024-11-14T02:46:08,434 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T02:46:08,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:08,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T02:46:08,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:08,520 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=136.42 KB heapSize=390.06 KB 2024-11-14T02:46:08,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:08,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:46:08,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:08,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:46:08,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:08,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:46:08,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T02:46:08,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145282da8fe01f40f6a276b7734edc2ede_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552368235/Put/seqid=0 2024-11-14T02:46:08,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:08,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742299_1475 (size=12154) 2024-11-14T02:46:08,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:08,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552428566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,575 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145282da8fe01f40f6a276b7734edc2ede_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145282da8fe01f40f6a276b7734edc2ede_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:08,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552428566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/ff0c6a726dd44b148d25ccf8a767b6c9, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:08,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/ff0c6a726dd44b148d25ccf8a767b6c9 is 175, key is test_row_0/A:col10/1731552368235/Put/seqid=0 2024-11-14T02:46:08,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552428573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552428575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552428576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742300_1476 (size=30955) 2024-11-14T02:46:08,614 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/ff0c6a726dd44b148d25ccf8a767b6c9 2024-11-14T02:46:08,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/edc595fb12104310a092f78282b94d6b is 50, key is test_row_0/B:col10/1731552368235/Put/seqid=0 2024-11-14T02:46:08,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742301_1477 (size=12001) 2024-11-14T02:46:08,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552428685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552428685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552428685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552428685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T02:46:08,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552428885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552428895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552428896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552428896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552428896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,047 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/edc595fb12104310a092f78282b94d6b 2024-11-14T02:46:09,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/5c014bf75f1241db9668ed345ff6b46b is 50, key is test_row_0/C:col10/1731552368235/Put/seqid=0 2024-11-14T02:46:09,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742302_1478 (size=12001) 2024-11-14T02:46:09,112 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/5c014bf75f1241db9668ed345ff6b46b 2024-11-14T02:46:09,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/ff0c6a726dd44b148d25ccf8a767b6c9 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/ff0c6a726dd44b148d25ccf8a767b6c9 2024-11-14T02:46:09,122 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/ff0c6a726dd44b148d25ccf8a767b6c9, entries=150, sequenceid=51, filesize=30.2 K 2024-11-14T02:46:09,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/edc595fb12104310a092f78282b94d6b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/edc595fb12104310a092f78282b94d6b 2024-11-14T02:46:09,128 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/edc595fb12104310a092f78282b94d6b, entries=150, sequenceid=51, filesize=11.7 K 2024-11-14T02:46:09,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/5c014bf75f1241db9668ed345ff6b46b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/5c014bf75f1241db9668ed345ff6b46b 2024-11-14T02:46:09,136 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/5c014bf75f1241db9668ed345ff6b46b, entries=150, sequenceid=51, filesize=11.7 K 2024-11-14T02:46:09,137 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~390.02 KB/399376, currentSize=46.96 KB/48090 for 672d111a762fbc594a3a447bdac7fae8 in 618ms, sequenceid=51, compaction requested=false 2024-11-14T02:46:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-14T02:46:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-14T02:46:09,143 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-14T02:46:09,143 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 929 msec 2024-11-14T02:46:09,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 944 msec 2024-11-14T02:46:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:09,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T02:46:09,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:09,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:09,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:09,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:09,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:09,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:09,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d8811f33b4ea4b8e82e9162476e58d46_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552369205/Put/seqid=0 2024-11-14T02:46:09,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742303_1479 (size=12154) 2024-11-14T02:46:09,223 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:09,230 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d8811f33b4ea4b8e82e9162476e58d46_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d8811f33b4ea4b8e82e9162476e58d46_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:09,231 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/213513e88d1542c29a636dca01794148, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:09,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/213513e88d1542c29a636dca01794148 is 175, key is test_row_0/A:col10/1731552369205/Put/seqid=0 2024-11-14T02:46:09,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742304_1480 (size=30955) 2024-11-14T02:46:09,244 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/213513e88d1542c29a636dca01794148 2024-11-14T02:46:09,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552429239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731552369220, max timestamp=1731552369246Num uniques 859; 2024-11-14T02:46:09,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:46:09,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731552369220, max timestamp=1731552369246Num uniques 859; 2024-11-14T02:46:09,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T02:46:09,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731552369220, max timestamp=1731552369246Num uniques 859; 2024-11-14T02:46:09,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T02:46:09,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/15ef18ce5e3b46ef9a6c0d7480657bc3 is 50, key is test_row_0/B:col10/1731552369205/Put/seqid=0 2024-11-14T02:46:09,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742305_1481 (size=12001) 2024-11-14T02:46:09,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/15ef18ce5e3b46ef9a6c0d7480657bc3 2024-11-14T02:46:09,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552429290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552429301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552429302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/97553bebb5ac4900b09e354e94d55c3b is 50, key is test_row_0/C:col10/1731552369205/Put/seqid=0 2024-11-14T02:46:09,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742306_1482 (size=12001) 2024-11-14T02:46:09,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/97553bebb5ac4900b09e354e94d55c3b 2024-11-14T02:46:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T02:46:09,345 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:09,346 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-14T02:46:09,347 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T02:46:09,348 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:09,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:09,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/213513e88d1542c29a636dca01794148 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/213513e88d1542c29a636dca01794148 2024-11-14T02:46:09,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/213513e88d1542c29a636dca01794148, entries=150, sequenceid=63, filesize=30.2 K 2024-11-14T02:46:09,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552429356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/15ef18ce5e3b46ef9a6c0d7480657bc3 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/15ef18ce5e3b46ef9a6c0d7480657bc3 2024-11-14T02:46:09,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/15ef18ce5e3b46ef9a6c0d7480657bc3, entries=150, sequenceid=63, filesize=11.7 K 2024-11-14T02:46:09,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/97553bebb5ac4900b09e354e94d55c3b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/97553bebb5ac4900b09e354e94d55c3b 2024-11-14T02:46:09,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/97553bebb5ac4900b09e354e94d55c3b, entries=150, sequenceid=63, filesize=11.7 K 2024-11-14T02:46:09,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=129.71 KB/132820 for 672d111a762fbc594a3a447bdac7fae8 in 168ms, sequenceid=63, compaction requested=true 2024-11-14T02:46:09,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:09,375 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:09,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:46:09,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:09,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:46:09,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:09,376 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:09,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:46:09,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:09,381 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:09,381 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:09,381 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/A is initiating minor compaction (all files) 2024-11-14T02:46:09,381 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/B is initiating minor compaction (all files) 2024-11-14T02:46:09,381 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/B in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:09,381 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/A in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:09,381 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/df554a3240454e0593af964f47297fe3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/ff0c6a726dd44b148d25ccf8a767b6c9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/213513e88d1542c29a636dca01794148] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=90.7 K 2024-11-14T02:46:09,381 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/c24927d1409a40748bf92bfe82685c97, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/edc595fb12104310a092f78282b94d6b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/15ef18ce5e3b46ef9a6c0d7480657bc3] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=35.2 K 2024-11-14T02:46:09,381 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:09,381 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/df554a3240454e0593af964f47297fe3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/ff0c6a726dd44b148d25ccf8a767b6c9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/213513e88d1542c29a636dca01794148] 2024-11-14T02:46:09,382 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting c24927d1409a40748bf92bfe82685c97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731552368169 2024-11-14T02:46:09,382 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting edc595fb12104310a092f78282b94d6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731552368231 2024-11-14T02:46:09,382 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting df554a3240454e0593af964f47297fe3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731552368169 2024-11-14T02:46:09,382 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff0c6a726dd44b148d25ccf8a767b6c9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731552368231 2024-11-14T02:46:09,383 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 213513e88d1542c29a636dca01794148, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731552368573 2024-11-14T02:46:09,383 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 15ef18ce5e3b46ef9a6c0d7480657bc3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731552368573 2024-11-14T02:46:09,393 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#B#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:09,393 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/d7a71e081c9b425baae4cdd04c2c3cba is 50, key is test_row_0/B:col10/1731552369205/Put/seqid=0 2024-11-14T02:46:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:09,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-11-14T02:46:09,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:09,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:46:09,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:09,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:46:09,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:09,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:46:09,405 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:09,422 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411141644b2aba9a6412bb78f21c885b2e1ca_672d111a762fbc594a3a447bdac7fae8 store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:09,425 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411141644b2aba9a6412bb78f21c885b2e1ca_672d111a762fbc594a3a447bdac7fae8, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:09,425 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141644b2aba9a6412bb78f21c885b2e1ca_672d111a762fbc594a3a447bdac7fae8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:09,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552429421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552429421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552429422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552429423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742307_1483 (size=12104) 2024-11-14T02:46:09,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ead9d05da0ce474298d97b1767343523_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552369399/Put/seqid=0 2024-11-14T02:46:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T02:46:09,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:09,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:09,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:09,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:09,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:09,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:09,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742308_1484 (size=4469) 2024-11-14T02:46:09,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742309_1485 (size=14594) 2024-11-14T02:46:09,528 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:09,537 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ead9d05da0ce474298d97b1767343523_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ead9d05da0ce474298d97b1767343523_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:09,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552429535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,538 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/b7a3ee6426994e65a2b595ac16be9d5c, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552429535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/b7a3ee6426994e65a2b595ac16be9d5c is 175, key is test_row_0/A:col10/1731552369399/Put/seqid=0 2024-11-14T02:46:09,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552429536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552429536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742310_1486 (size=39549) 2024-11-14T02:46:09,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552429566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:09,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:09,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:09,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:09,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:09,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T02:46:09,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552429745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552429746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552429746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552429746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:09,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:09,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:09,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:09,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:09,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:09,860 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/d7a71e081c9b425baae4cdd04c2c3cba as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/d7a71e081c9b425baae4cdd04c2c3cba 2024-11-14T02:46:09,867 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/B of 672d111a762fbc594a3a447bdac7fae8 into d7a71e081c9b425baae4cdd04c2c3cba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:09,867 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:09,867 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/B, priority=13, startTime=1731552369376; duration=0sec 2024-11-14T02:46:09,867 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:09,867 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:B 2024-11-14T02:46:09,867 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:09,869 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:09,869 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/C is initiating minor compaction (all files) 2024-11-14T02:46:09,869 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/C in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:09,869 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/fb4039ebdc87407b951a8daf2ced6df1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/5c014bf75f1241db9668ed345ff6b46b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/97553bebb5ac4900b09e354e94d55c3b] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=35.2 K 2024-11-14T02:46:09,871 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting fb4039ebdc87407b951a8daf2ced6df1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731552368169 2024-11-14T02:46:09,871 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c014bf75f1241db9668ed345ff6b46b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731552368231 2024-11-14T02:46:09,872 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 97553bebb5ac4900b09e354e94d55c3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731552368573 2024-11-14T02:46:09,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552429875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:09,881 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#C#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:09,882 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/1a2115044fa7440ba687dfa52487ff83 is 50, key is test_row_0/C:col10/1731552369205/Put/seqid=0 2024-11-14T02:46:09,907 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#A#compaction#411 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:09,907 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/0c74967c7e394b5389bec3c5598ab920 is 175, key is test_row_0/A:col10/1731552369205/Put/seqid=0 2024-11-14T02:46:09,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742311_1487 (size=12104) 2024-11-14T02:46:09,927 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/1a2115044fa7440ba687dfa52487ff83 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/1a2115044fa7440ba687dfa52487ff83 2024-11-14T02:46:09,932 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/C of 672d111a762fbc594a3a447bdac7fae8 into 1a2115044fa7440ba687dfa52487ff83(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:09,932 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:09,932 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/C, priority=13, startTime=1731552369376; duration=0sec 2024-11-14T02:46:09,932 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:09,932 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:C 2024-11-14T02:46:09,961 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/b7a3ee6426994e65a2b595ac16be9d5c 2024-11-14T02:46:09,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:09,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:09,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:09,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:09,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:09,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:09,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742312_1488 (size=31058) 2024-11-14T02:46:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T02:46:09,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/db08164ea5164d6fbe7c18fa146469f8 is 50, key is test_row_0/B:col10/1731552369399/Put/seqid=0 2024-11-14T02:46:10,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742313_1489 (size=12001) 2024-11-14T02:46:10,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552430056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:10,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:10,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552430056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:10,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552430056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552430056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:10,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:10,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:10,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:10,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:10,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:10,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:10,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:10,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,345 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T02:46:10,377 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/0c74967c7e394b5389bec3c5598ab920 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/0c74967c7e394b5389bec3c5598ab920 2024-11-14T02:46:10,382 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/A of 672d111a762fbc594a3a447bdac7fae8 into 0c74967c7e394b5389bec3c5598ab920(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-14T02:46:10,382 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:10,382 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/A, priority=13, startTime=1731552369375; duration=1sec 2024-11-14T02:46:10,382 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:10,382 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:A 2024-11-14T02:46:10,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552430385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:10,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:10,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:10,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/db08164ea5164d6fbe7c18fa146469f8 2024-11-14T02:46:10,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/41fa531b3fd64e13bb62f8da67c7d5b0 is 50, key is test_row_0/C:col10/1731552369399/Put/seqid=0 2024-11-14T02:46:10,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742314_1490 (size=12001) 2024-11-14T02:46:10,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/41fa531b3fd64e13bb62f8da67c7d5b0 2024-11-14T02:46:10,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/b7a3ee6426994e65a2b595ac16be9d5c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/b7a3ee6426994e65a2b595ac16be9d5c 2024-11-14T02:46:10,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/b7a3ee6426994e65a2b595ac16be9d5c, entries=200, sequenceid=99, filesize=38.6 K 2024-11-14T02:46:10,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/db08164ea5164d6fbe7c18fa146469f8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/db08164ea5164d6fbe7c18fa146469f8 2024-11-14T02:46:10,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/db08164ea5164d6fbe7c18fa146469f8, entries=150, sequenceid=99, filesize=11.7 K 2024-11-14T02:46:10,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/41fa531b3fd64e13bb62f8da67c7d5b0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/41fa531b3fd64e13bb62f8da67c7d5b0 2024-11-14T02:46:10,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/41fa531b3fd64e13bb62f8da67c7d5b0, entries=150, sequenceid=99, filesize=11.7 K 2024-11-14T02:46:10,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=46.96 KB/48090 for 672d111a762fbc594a3a447bdac7fae8 in 1081ms, sequenceid=99, compaction requested=false 2024-11-14T02:46:10,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T02:46:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T02:46:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:10,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:10,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:10,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:10,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:10,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111443840b8e702f4d05a7e42f339dd7e071_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_1/A:col10/1731552370571/Put/seqid=0 2024-11-14T02:46:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:10,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:10,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742315_1491 (size=12154) 2024-11-14T02:46:10,587 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,591 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111443840b8e702f4d05a7e42f339dd7e071_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111443840b8e702f4d05a7e42f339dd7e071_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,592 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/e38fec9b634d4cfb92e1a86e2166509e, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/e38fec9b634d4cfb92e1a86e2166509e is 175, key is test_row_1/A:col10/1731552370571/Put/seqid=0 2024-11-14T02:46:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:10,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742316_1492 (size=30951) 2024-11-14T02:46:10,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552370646, max timestamp=1731552370654Num uniques 859; 2024-11-14T02:46:10,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:46:10,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552370646, max timestamp=1731552370654Num uniques 859; 2024-11-14T02:46:10,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:46:10,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731552370646, max timestamp=1731552370654Num uniques 859; 2024-11-14T02:46:10,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e5dc8b5cf5e9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T02:46:10,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552430679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:10,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:10,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552430681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552430681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:10,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552430682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:10,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:10,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:10,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552430795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552430795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:10,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:10,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552430795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552430795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:10,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:10,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,008 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/e38fec9b634d4cfb92e1a86e2166509e 2024-11-14T02:46:11,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552431006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552431006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:11,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552431006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552431006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/87a320849bd3436c89fe6e7bf22ecaf5 is 50, key is test_row_1/B:col10/1731552370571/Put/seqid=0 2024-11-14T02:46:11,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742317_1493 (size=9657) 2024-11-14T02:46:11,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:11,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:11,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552431315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:11,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552431315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552431316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552431316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:11,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:11,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552431405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:11,409 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in 672d111a762fbc594a3a447bdac7fae8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:46:11,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/87a320849bd3436c89fe6e7bf22ecaf5 2024-11-14T02:46:11,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/680eb7c574c4456d8b6cfabc82085705 is 50, key is test_row_1/C:col10/1731552370571/Put/seqid=0 2024-11-14T02:46:11,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742318_1494 (size=9657) 2024-11-14T02:46:11,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:11,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:11,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T02:46:11,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:11,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:11,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:11,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:11,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:11,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552431826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:11,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:11,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552431826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:11,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552431826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552431835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:11,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/680eb7c574c4456d8b6cfabc82085705 2024-11-14T02:46:11,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/e38fec9b634d4cfb92e1a86e2166509e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/e38fec9b634d4cfb92e1a86e2166509e 2024-11-14T02:46:11,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/e38fec9b634d4cfb92e1a86e2166509e, entries=150, sequenceid=113, filesize=30.2 K 2024-11-14T02:46:11,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/87a320849bd3436c89fe6e7bf22ecaf5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/87a320849bd3436c89fe6e7bf22ecaf5 2024-11-14T02:46:11,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/87a320849bd3436c89fe6e7bf22ecaf5, entries=100, sequenceid=113, filesize=9.4 K 2024-11-14T02:46:11,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/680eb7c574c4456d8b6cfabc82085705 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/680eb7c574c4456d8b6cfabc82085705 2024-11-14T02:46:11,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/680eb7c574c4456d8b6cfabc82085705, entries=100, sequenceid=113, filesize=9.4 K 2024-11-14T02:46:11,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 672d111a762fbc594a3a447bdac7fae8 in 1327ms, sequenceid=113, compaction requested=true 2024-11-14T02:46:11,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:11,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:46:11,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:11,900 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:11,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:46:11,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:11,900 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:11,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:46:11,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:11,901 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:11,901 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101558 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:11,901 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/B is initiating minor compaction (all files) 2024-11-14T02:46:11,901 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/A is initiating minor compaction (all files) 2024-11-14T02:46:11,901 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/A in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,901 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/B in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,901 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/d7a71e081c9b425baae4cdd04c2c3cba, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/db08164ea5164d6fbe7c18fa146469f8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/87a320849bd3436c89fe6e7bf22ecaf5] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=33.0 K 2024-11-14T02:46:11,901 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/0c74967c7e394b5389bec3c5598ab920, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/b7a3ee6426994e65a2b595ac16be9d5c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/e38fec9b634d4cfb92e1a86e2166509e] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=99.2 K 2024-11-14T02:46:11,902 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,902 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/0c74967c7e394b5389bec3c5598ab920, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/b7a3ee6426994e65a2b595ac16be9d5c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/e38fec9b634d4cfb92e1a86e2166509e] 2024-11-14T02:46:11,902 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting d7a71e081c9b425baae4cdd04c2c3cba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731552368573 2024-11-14T02:46:11,902 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c74967c7e394b5389bec3c5598ab920, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731552368573 2024-11-14T02:46:11,902 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting db08164ea5164d6fbe7c18fa146469f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731552369288 2024-11-14T02:46:11,902 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 87a320849bd3436c89fe6e7bf22ecaf5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731552369421 2024-11-14T02:46:11,902 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7a3ee6426994e65a2b595ac16be9d5c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731552369288 2024-11-14T02:46:11,903 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting e38fec9b634d4cfb92e1a86e2166509e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731552369420 2024-11-14T02:46:11,914 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:11,914 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#B#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:11,915 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/5494be801b6a4aa790c29035dc41621a is 50, key is test_row_0/B:col10/1731552369399/Put/seqid=0 2024-11-14T02:46:11,925 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114558134c26a7f46ef962bac7343866c5c_672d111a762fbc594a3a447bdac7fae8 store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:11,927 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114558134c26a7f46ef962bac7343866c5c_672d111a762fbc594a3a447bdac7fae8, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:11,927 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114558134c26a7f46ef962bac7343866c5c_672d111a762fbc594a3a447bdac7fae8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:11,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742320_1496 (size=4469) 2024-11-14T02:46:11,933 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#A#compaction#420 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:11,933 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/0931933e9fbc4e6c8217333c4d9e9628 is 175, key is test_row_0/A:col10/1731552369399/Put/seqid=0 2024-11-14T02:46:11,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742319_1495 (size=12207) 2024-11-14T02:46:11,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742321_1497 (size=31268) 2024-11-14T02:46:11,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T02:46:11,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:11,962 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-14T02:46:11,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:11,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:46:11,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:11,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:46:11,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:11,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:46:11,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fb5291d9958248e2bf3b223a59ac5f01_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552370677/Put/seqid=0 2024-11-14T02:46:12,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742322_1498 (size=12304) 2024-11-14T02:46:12,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:12,015 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fb5291d9958248e2bf3b223a59ac5f01_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fb5291d9958248e2bf3b223a59ac5f01_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:12,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/d9cc1447410c44afacc694b0616e5a5f, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/d9cc1447410c44afacc694b0616e5a5f is 175, key is test_row_0/A:col10/1731552370677/Put/seqid=0 2024-11-14T02:46:12,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742323_1499 (size=31105) 2024-11-14T02:46:12,022 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/d9cc1447410c44afacc694b0616e5a5f 2024-11-14T02:46:12,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/95d287bd82f049f49c02d4ba9e118fa2 is 50, key is test_row_0/B:col10/1731552370677/Put/seqid=0 2024-11-14T02:46:12,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742324_1500 (size=12151) 2024-11-14T02:46:12,347 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/5494be801b6a4aa790c29035dc41621a as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/5494be801b6a4aa790c29035dc41621a 2024-11-14T02:46:12,351 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/B of 672d111a762fbc594a3a447bdac7fae8 into 5494be801b6a4aa790c29035dc41621a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:12,351 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:12,351 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/B, priority=13, startTime=1731552371900; duration=0sec 2024-11-14T02:46:12,351 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:12,351 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:B 2024-11-14T02:46:12,351 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:12,352 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:12,352 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/C is initiating minor compaction (all files) 2024-11-14T02:46:12,352 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/C in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:12,352 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/1a2115044fa7440ba687dfa52487ff83, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/41fa531b3fd64e13bb62f8da67c7d5b0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/680eb7c574c4456d8b6cfabc82085705] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=33.0 K 2024-11-14T02:46:12,352 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a2115044fa7440ba687dfa52487ff83, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731552368573 2024-11-14T02:46:12,353 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 41fa531b3fd64e13bb62f8da67c7d5b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731552369288 2024-11-14T02:46:12,353 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 680eb7c574c4456d8b6cfabc82085705, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731552369421 2024-11-14T02:46:12,358 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#C#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:12,359 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/a304f04d82374f4c816d9cd00507e8e3 is 50, key is test_row_0/C:col10/1731552369399/Put/seqid=0 2024-11-14T02:46:12,361 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/0931933e9fbc4e6c8217333c4d9e9628 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/0931933e9fbc4e6c8217333c4d9e9628 2024-11-14T02:46:12,367 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/A of 672d111a762fbc594a3a447bdac7fae8 into 0931933e9fbc4e6c8217333c4d9e9628(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:12,367 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:12,367 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/A, priority=13, startTime=1731552371900; duration=0sec 2024-11-14T02:46:12,367 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:12,367 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:A 2024-11-14T02:46:12,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742325_1501 (size=12207) 2024-11-14T02:46:12,435 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/95d287bd82f049f49c02d4ba9e118fa2 2024-11-14T02:46:12,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/63c80f872a1f40a7a827958b7c7013f0 is 50, key is test_row_0/C:col10/1731552370677/Put/seqid=0 2024-11-14T02:46:12,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742326_1502 (size=12151) 2024-11-14T02:46:12,804 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/a304f04d82374f4c816d9cd00507e8e3 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/a304f04d82374f4c816d9cd00507e8e3 2024-11-14T02:46:12,809 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/C of 672d111a762fbc594a3a447bdac7fae8 into a304f04d82374f4c816d9cd00507e8e3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:12,809 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:12,809 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/C, priority=13, startTime=1731552371900; duration=0sec 2024-11-14T02:46:12,809 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:12,809 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:C 2024-11-14T02:46:12,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:12,850 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/63c80f872a1f40a7a827958b7c7013f0 2024-11-14T02:46:12,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/d9cc1447410c44afacc694b0616e5a5f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/d9cc1447410c44afacc694b0616e5a5f 2024-11-14T02:46:12,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552432851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:12,858 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,858 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,858 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:12,858 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/d9cc1447410c44afacc694b0616e5a5f, entries=150, sequenceid=148, filesize=30.4 K 2024-11-14T02:46:12,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552432855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:12,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:12,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/95d287bd82f049f49c02d4ba9e118fa2 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/95d287bd82f049f49c02d4ba9e118fa2 2024-11-14T02:46:12,862 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/95d287bd82f049f49c02d4ba9e118fa2, entries=150, sequenceid=148, filesize=11.9 K 2024-11-14T02:46:12,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/63c80f872a1f40a7a827958b7c7013f0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/63c80f872a1f40a7a827958b7c7013f0 2024-11-14T02:46:12,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552432857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:12,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552432858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:12,867 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/63c80f872a1f40a7a827958b7c7013f0, entries=150, sequenceid=148, filesize=11.9 K 2024-11-14T02:46:12,868 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 672d111a762fbc594a3a447bdac7fae8 in 906ms, sequenceid=148, compaction requested=false 2024-11-14T02:46:12,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:12,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:12,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-14T02:46:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-14T02:46:12,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-14T02:46:12,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5210 sec 2024-11-14T02:46:12,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 3.5250 sec 2024-11-14T02:46:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:12,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T02:46:12,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:12,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:12,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:12,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:12,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:12,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:12,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b10a6e325cfd44ab9df6f02dc6dfe659_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552372968/Put/seqid=0 2024-11-14T02:46:12,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742327_1503 (size=17284) 2024-11-14T02:46:13,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552432994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552432997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552432999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552433000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552433105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552433115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552433115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552433115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552433315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,319 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,319 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,319 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552433325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552433325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552433326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,392 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:13,400 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b10a6e325cfd44ab9df6f02dc6dfe659_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b10a6e325cfd44ab9df6f02dc6dfe659_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:13,401 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/ec69640f4c0940c5ab17c66133c41287, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:13,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/ec69640f4c0940c5ab17c66133c41287 is 175, key is test_row_0/A:col10/1731552372968/Put/seqid=0 2024-11-14T02:46:13,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742328_1504 (size=48389) 2024-11-14T02:46:13,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552433415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,420 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in 672d111a762fbc594a3a447bdac7fae8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:46:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T02:46:13,525 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:13,525 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-14T02:46:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T02:46:13,528 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:13,529 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:13,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:13,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552433625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T02:46:13,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552433635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552433635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:13,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552433635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:13,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:13,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:13,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T02:46:13,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:13,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:13,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:13,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:13,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:13,811 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/ec69640f4c0940c5ab17c66133c41287 2024-11-14T02:46:13,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/a1752ecb0bc9478f8daa92054ec83e9b is 50, key is test_row_0/B:col10/1731552372968/Put/seqid=0 2024-11-14T02:46:13,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742329_1505 (size=12151) 2024-11-14T02:46:13,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T02:46:13,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:13,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:13,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:13,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:13,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T02:46:13,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T02:46:13,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:13,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:13,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:13,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:13,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:14,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552434135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:14,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T02:46:14,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:14,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:14,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:14,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:14,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:14,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552434145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:14,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552434145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:14,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552434145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T02:46:14,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/a1752ecb0bc9478f8daa92054ec83e9b 2024-11-14T02:46:14,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/884e80c65c3543c094125741f814bb31 is 50, key is test_row_0/C:col10/1731552372968/Put/seqid=0 2024-11-14T02:46:14,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742330_1506 (size=12151) 2024-11-14T02:46:14,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/884e80c65c3543c094125741f814bb31 2024-11-14T02:46:14,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/ec69640f4c0940c5ab17c66133c41287 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/ec69640f4c0940c5ab17c66133c41287 2024-11-14T02:46:14,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/ec69640f4c0940c5ab17c66133c41287, entries=250, sequenceid=165, filesize=47.3 K 2024-11-14T02:46:14,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/a1752ecb0bc9478f8daa92054ec83e9b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a1752ecb0bc9478f8daa92054ec83e9b 2024-11-14T02:46:14,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a1752ecb0bc9478f8daa92054ec83e9b, entries=150, sequenceid=165, filesize=11.9 K 2024-11-14T02:46:14,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/884e80c65c3543c094125741f814bb31 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/884e80c65c3543c094125741f814bb31 2024-11-14T02:46:14,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/884e80c65c3543c094125741f814bb31, entries=150, sequenceid=165, filesize=11.9 K 2024-11-14T02:46:14,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 672d111a762fbc594a3a447bdac7fae8 in 1279ms, sequenceid=165, compaction requested=true 2024-11-14T02:46:14,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:14,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:46:14,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:14,248 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:14,248 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:14,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:46:14,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:14,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:46:14,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:14,252 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:14,252 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/A is initiating minor compaction (all files) 2024-11-14T02:46:14,253 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/A in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:14,253 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:14,253 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/0931933e9fbc4e6c8217333c4d9e9628, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/d9cc1447410c44afacc694b0616e5a5f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/ec69640f4c0940c5ab17c66133c41287] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=108.2 K 2024-11-14T02:46:14,253 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/B is initiating minor compaction (all files) 2024-11-14T02:46:14,253 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:14,253 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/B in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:14,253 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/0931933e9fbc4e6c8217333c4d9e9628, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/d9cc1447410c44afacc694b0616e5a5f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/ec69640f4c0940c5ab17c66133c41287] 2024-11-14T02:46:14,253 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/5494be801b6a4aa790c29035dc41621a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/95d287bd82f049f49c02d4ba9e118fa2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a1752ecb0bc9478f8daa92054ec83e9b] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=35.7 K 2024-11-14T02:46:14,254 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 5494be801b6a4aa790c29035dc41621a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731552369399 2024-11-14T02:46:14,254 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0931933e9fbc4e6c8217333c4d9e9628, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731552369399 2024-11-14T02:46:14,254 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 95d287bd82f049f49c02d4ba9e118fa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731552370666 2024-11-14T02:46:14,254 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9cc1447410c44afacc694b0616e5a5f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731552370666 2024-11-14T02:46:14,254 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting a1752ecb0bc9478f8daa92054ec83e9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731552372856 2024-11-14T02:46:14,254 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec69640f4c0940c5ab17c66133c41287, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731552372848 2024-11-14T02:46:14,259 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:14,261 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411148cb41d9a691946a88535b981f42b2154_672d111a762fbc594a3a447bdac7fae8 store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:14,261 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#B#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:14,261 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/771d400fd27d4480881872850b2a7f70 is 50, key is test_row_0/B:col10/1731552372968/Put/seqid=0 2024-11-14T02:46:14,263 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411148cb41d9a691946a88535b981f42b2154_672d111a762fbc594a3a447bdac7fae8, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:14,263 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148cb41d9a691946a88535b981f42b2154_672d111a762fbc594a3a447bdac7fae8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:14,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742332_1508 (size=4469) 2024-11-14T02:46:14,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742331_1507 (size=12459) 2024-11-14T02:46:14,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T02:46:14,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:14,292 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T02:46:14,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:14,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:14,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:14,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:14,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:14,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:14,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c19075ee44e74291862cc20eb9928eee_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552372998/Put/seqid=0 2024-11-14T02:46:14,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742333_1509 (size=12304) 2024-11-14T02:46:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T02:46:14,680 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#A#compaction#428 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:14,680 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/6321bdfd2574436aba08f2386a55b8ee is 175, key is test_row_0/A:col10/1731552372968/Put/seqid=0 2024-11-14T02:46:14,687 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/771d400fd27d4480881872850b2a7f70 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/771d400fd27d4480881872850b2a7f70 2024-11-14T02:46:14,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742334_1510 (size=31413) 2024-11-14T02:46:14,695 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/B of 672d111a762fbc594a3a447bdac7fae8 into 771d400fd27d4480881872850b2a7f70(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:14,695 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:14,696 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/6321bdfd2574436aba08f2386a55b8ee as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/6321bdfd2574436aba08f2386a55b8ee 2024-11-14T02:46:14,696 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/B, priority=13, startTime=1731552374248; duration=0sec 2024-11-14T02:46:14,696 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:14,696 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:B 2024-11-14T02:46:14,696 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:14,698 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:14,698 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/C is initiating minor compaction (all files) 2024-11-14T02:46:14,698 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/C in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:14,698 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/a304f04d82374f4c816d9cd00507e8e3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/63c80f872a1f40a7a827958b7c7013f0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/884e80c65c3543c094125741f814bb31] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=35.7 K 2024-11-14T02:46:14,699 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting a304f04d82374f4c816d9cd00507e8e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731552369399 2024-11-14T02:46:14,700 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 63c80f872a1f40a7a827958b7c7013f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731552370666 2024-11-14T02:46:14,700 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 884e80c65c3543c094125741f814bb31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731552372856 2024-11-14T02:46:14,701 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/A of 672d111a762fbc594a3a447bdac7fae8 into 6321bdfd2574436aba08f2386a55b8ee(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:14,702 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:14,702 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/A, priority=13, startTime=1731552374248; duration=0sec 2024-11-14T02:46:14,702 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:14,702 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:A 2024-11-14T02:46:14,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:14,708 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c19075ee44e74291862cc20eb9928eee_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c19075ee44e74291862cc20eb9928eee_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:14,708 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#C#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:14,709 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/3e42334a8d944d9cb37a63ef67607e80 is 50, key is test_row_0/C:col10/1731552372968/Put/seqid=0 2024-11-14T02:46:14,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/1e8100c2054c4ef7a6430d89ef82e84a, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:14,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/1e8100c2054c4ef7a6430d89ef82e84a is 175, key is test_row_0/A:col10/1731552372998/Put/seqid=0 2024-11-14T02:46:14,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742336_1512 (size=12459) 2024-11-14T02:46:14,743 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/3e42334a8d944d9cb37a63ef67607e80 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/3e42334a8d944d9cb37a63ef67607e80 2024-11-14T02:46:14,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742335_1511 (size=31105) 2024-11-14T02:46:14,749 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/C of 672d111a762fbc594a3a447bdac7fae8 into 3e42334a8d944d9cb37a63ef67607e80(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:14,749 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:14,749 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/C, priority=13, startTime=1731552374248; duration=0sec 2024-11-14T02:46:14,749 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:14,749 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:C 2024-11-14T02:46:15,146 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/1e8100c2054c4ef7a6430d89ef82e84a 2024-11-14T02:46:15,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:15,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/cd6b56e37ebb47d8aa278698e0900b49 is 50, key is test_row_0/B:col10/1731552372998/Put/seqid=0 2024-11-14T02:46:15,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552435167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552435171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742337_1513 (size=12151) 2024-11-14T02:46:15,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552435173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552435173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552435275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552435285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552435285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552435285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552435486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552435495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552435495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552435495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,577 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/cd6b56e37ebb47d8aa278698e0900b49 2024-11-14T02:46:15,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/66bef4ec5f284b22abea2cd033413525 is 50, key is test_row_0/C:col10/1731552372998/Put/seqid=0 2024-11-14T02:46:15,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742338_1514 (size=12151) 2024-11-14T02:46:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T02:46:15,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552435795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552435805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552435805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552435805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:15,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:15,996 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/66bef4ec5f284b22abea2cd033413525 2024-11-14T02:46:16,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/1e8100c2054c4ef7a6430d89ef82e84a as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/1e8100c2054c4ef7a6430d89ef82e84a 2024-11-14T02:46:16,009 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/1e8100c2054c4ef7a6430d89ef82e84a, entries=150, sequenceid=183, filesize=30.4 K 2024-11-14T02:46:16,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/cd6b56e37ebb47d8aa278698e0900b49 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/cd6b56e37ebb47d8aa278698e0900b49 2024-11-14T02:46:16,020 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/cd6b56e37ebb47d8aa278698e0900b49, entries=150, sequenceid=183, filesize=11.9 K 2024-11-14T02:46:16,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/66bef4ec5f284b22abea2cd033413525 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/66bef4ec5f284b22abea2cd033413525 2024-11-14T02:46:16,026 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/66bef4ec5f284b22abea2cd033413525, entries=150, sequenceid=183, filesize=11.9 K 2024-11-14T02:46:16,026 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 672d111a762fbc594a3a447bdac7fae8 in 1734ms, sequenceid=183, compaction requested=false 2024-11-14T02:46:16,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:16,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:16,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-14T02:46:16,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-14T02:46:16,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-14T02:46:16,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4990 sec 2024-11-14T02:46:16,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.5040 sec 2024-11-14T02:46:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:16,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T02:46:16,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:16,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:16,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:16,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:16,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:16,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:16,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114170ae0d28746408e84adfb6592aa3a23_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552375171/Put/seqid=0 2024-11-14T02:46:16,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552436336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552436338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552436341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:16,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552436343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:16,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742339_1515 (size=12304) 2024-11-14T02:46:16,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552436446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552436446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:16,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552436455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:16,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552436455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552436656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:16,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552436656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:16,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552436665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:16,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:16,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552436665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:16,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:16,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:16,772 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,778 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114170ae0d28746408e84adfb6592aa3a23_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114170ae0d28746408e84adfb6592aa3a23_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:16,779 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/99c4031b3a73483d905a75efa4ae0a28, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:16,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/99c4031b3a73483d905a75efa4ae0a28 is 175, key is test_row_0/A:col10/1731552375171/Put/seqid=0 2024-11-14T02:46:16,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742340_1516 (size=31105) 2024-11-14T02:46:16,825 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/99c4031b3a73483d905a75efa4ae0a28 2024-11-14T02:46:16,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/e682a9fef073406bb91e99a525895359 is 50, key is test_row_0/B:col10/1731552375171/Put/seqid=0 2024-11-14T02:46:16,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742341_1517 (size=12151) 2024-11-14T02:46:16,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/e682a9fef073406bb91e99a525895359 2024-11-14T02:46:16,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/fe19e10b40cc4c928e78b28597dafbbb is 50, key is test_row_0/C:col10/1731552375171/Put/seqid=0 2024-11-14T02:46:16,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742342_1518 (size=12151) 2024-11-14T02:46:16,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/fe19e10b40cc4c928e78b28597dafbbb 2024-11-14T02:46:16,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/99c4031b3a73483d905a75efa4ae0a28 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/99c4031b3a73483d905a75efa4ae0a28 2024-11-14T02:46:16,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/99c4031b3a73483d905a75efa4ae0a28, entries=150, sequenceid=202, filesize=30.4 K 2024-11-14T02:46:16,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/e682a9fef073406bb91e99a525895359 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/e682a9fef073406bb91e99a525895359 2024-11-14T02:46:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/e682a9fef073406bb91e99a525895359, entries=150, sequenceid=202, filesize=11.9 K 2024-11-14T02:46:16,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/fe19e10b40cc4c928e78b28597dafbbb as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/fe19e10b40cc4c928e78b28597dafbbb 2024-11-14T02:46:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/fe19e10b40cc4c928e78b28597dafbbb, entries=150, sequenceid=202, filesize=11.9 K 2024-11-14T02:46:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 672d111a762fbc594a3a447bdac7fae8 in 647ms, sequenceid=202, compaction requested=true 2024-11-14T02:46:16,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:16,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:46:16,956 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:16,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:16,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:46:16,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:16,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:46:16,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:16,956 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,957 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:16,957 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/A is initiating minor compaction (all files) 2024-11-14T02:46:16,957 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/A in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:16,957 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/6321bdfd2574436aba08f2386a55b8ee, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/1e8100c2054c4ef7a6430d89ef82e84a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/99c4031b3a73483d905a75efa4ae0a28] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=91.4 K 2024-11-14T02:46:16,957 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:16,957 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/6321bdfd2574436aba08f2386a55b8ee, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/1e8100c2054c4ef7a6430d89ef82e84a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/99c4031b3a73483d905a75efa4ae0a28] 2024-11-14T02:46:16,957 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:16,958 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/B is initiating minor compaction (all files) 2024-11-14T02:46:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,958 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/B in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:16,958 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/771d400fd27d4480881872850b2a7f70, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/cd6b56e37ebb47d8aa278698e0900b49, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/e682a9fef073406bb91e99a525895359] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=35.9 K 2024-11-14T02:46:16,958 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6321bdfd2574436aba08f2386a55b8ee, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731552372856 2024-11-14T02:46:16,958 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 771d400fd27d4480881872850b2a7f70, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731552372856 2024-11-14T02:46:16,958 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e8100c2054c4ef7a6430d89ef82e84a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731552372989 2024-11-14T02:46:16,958 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting cd6b56e37ebb47d8aa278698e0900b49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731552372989 2024-11-14T02:46:16,958 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99c4031b3a73483d905a75efa4ae0a28, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731552375165 2024-11-14T02:46:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,959 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting e682a9fef073406bb91e99a525895359, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731552375165 2024-11-14T02:46:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,971 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#B#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:16,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T02:46:16,971 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/f42c02b6276442209ec25053993db406 is 50, key is test_row_0/B:col10/1731552375171/Put/seqid=0 2024-11-14T02:46:16,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:16,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:16,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:16,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:16,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:16,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,982 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:16,986 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114ea4faa8a774f4724afc1513dfdceb2cb_672d111a762fbc594a3a447bdac7fae8 store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:16,989 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114ea4faa8a774f4724afc1513dfdceb2cb_672d111a762fbc594a3a447bdac7fae8, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:16,989 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ea4faa8a774f4724afc1513dfdceb2cb_672d111a762fbc594a3a447bdac7fae8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:17,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bbc00af0dc94405184253f8c9a3249be_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552376969/Put/seqid=0 2024-11-14T02:46:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:17,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552437014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552437020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552437023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552437038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742343_1519 (size=12561) 2024-11-14T02:46:17,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742344_1520 (size=4469) 2024-11-14T02:46:17,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742345_1521 (size=12304) 2024-11-14T02:46:17,067 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:17,072 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bbc00af0dc94405184253f8c9a3249be_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bbc00af0dc94405184253f8c9a3249be_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:17,076 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/4ea2e70138a8472db755ad99c58c4d14, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:17,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/4ea2e70138a8472db755ad99c58c4d14 is 175, key is test_row_0/A:col10/1731552376969/Put/seqid=0 2024-11-14T02:46:17,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742346_1522 (size=31101) 2024-11-14T02:46:17,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552437135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552437136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552437155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552437155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552437345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552437345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552437365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552437365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,456 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/f42c02b6276442209ec25053993db406 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/f42c02b6276442209ec25053993db406 2024-11-14T02:46:17,460 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#A#compaction#438 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:17,461 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/cc094a2f5717472a983572a263d64f38 is 175, key is test_row_0/A:col10/1731552375171/Put/seqid=0 2024-11-14T02:46:17,463 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/B of 672d111a762fbc594a3a447bdac7fae8 into f42c02b6276442209ec25053993db406(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:17,463 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:17,463 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/B, priority=13, startTime=1731552376956; duration=0sec 2024-11-14T02:46:17,463 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:17,463 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:B 2024-11-14T02:46:17,464 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:17,466 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:17,466 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/C is initiating minor compaction (all files) 2024-11-14T02:46:17,466 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/C in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:17,466 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/3e42334a8d944d9cb37a63ef67607e80, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/66bef4ec5f284b22abea2cd033413525, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/fe19e10b40cc4c928e78b28597dafbbb] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=35.9 K 2024-11-14T02:46:17,466 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e42334a8d944d9cb37a63ef67607e80, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731552372856 2024-11-14T02:46:17,467 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 66bef4ec5f284b22abea2cd033413525, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731552372989 2024-11-14T02:46:17,467 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting fe19e10b40cc4c928e78b28597dafbbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731552375165 2024-11-14T02:46:17,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552437465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,468 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_1' in 672d111a762fbc594a3a447bdac7fae8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:46:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,487 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/4ea2e70138a8472db755ad99c58c4d14 2024-11-14T02:46:17,495 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#C#compaction#440 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:17,497 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/22a01da3a4824ceb84703ccdf3c0653b is 50, key is test_row_0/C:col10/1731552375171/Put/seqid=0 2024-11-14T02:46:17,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742347_1523 (size=31515) 2024-11-14T02:46:17,507 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/cc094a2f5717472a983572a263d64f38 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/cc094a2f5717472a983572a263d64f38 2024-11-14T02:46:17,518 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/A of 672d111a762fbc594a3a447bdac7fae8 into cc094a2f5717472a983572a263d64f38(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:17,518 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:17,518 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/A, priority=13, startTime=1731552376956; duration=0sec 2024-11-14T02:46:17,518 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:17,519 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:A 2024-11-14T02:46:17,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/9be3e2576ac24e01830a5fe26e42d34c is 50, key is test_row_0/B:col10/1731552376969/Put/seqid=0 2024-11-14T02:46:17,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742348_1524 (size=12561) 2024-11-14T02:46:17,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742349_1525 (size=9757) 2024-11-14T02:46:17,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/9be3e2576ac24e01830a5fe26e42d34c 2024-11-14T02:46:17,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/77b511beecb444289a18e2f1c3981290 is 50, key is test_row_0/C:col10/1731552376969/Put/seqid=0 2024-11-14T02:46:17,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552437655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552437655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742350_1526 (size=9757) 2024-11-14T02:46:17,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/77b511beecb444289a18e2f1c3981290 2024-11-14T02:46:17,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/4ea2e70138a8472db755ad99c58c4d14 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/4ea2e70138a8472db755ad99c58c4d14 2024-11-14T02:46:17,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/4ea2e70138a8472db755ad99c58c4d14, entries=150, sequenceid=219, filesize=30.4 K 2024-11-14T02:46:17,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/9be3e2576ac24e01830a5fe26e42d34c as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/9be3e2576ac24e01830a5fe26e42d34c 2024-11-14T02:46:17,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552437676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552437676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:17,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/9be3e2576ac24e01830a5fe26e42d34c, entries=100, sequenceid=219, filesize=9.5 K 2024-11-14T02:46:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T02:46:17,686 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:17,686 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-14T02:46:17,688 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T02:46:17,688 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:17,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:17,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/77b511beecb444289a18e2f1c3981290 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/77b511beecb444289a18e2f1c3981290 2024-11-14T02:46:17,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/77b511beecb444289a18e2f1c3981290, entries=100, sequenceid=219, filesize=9.5 K 2024-11-14T02:46:17,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 672d111a762fbc594a3a447bdac7fae8 in 724ms, sequenceid=219, compaction requested=false 2024-11-14T02:46:17,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T02:46:17,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T02:46:17,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:17,842 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T02:46:17,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:17,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:17,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:17,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:17,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:17,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:17,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144df6e08b582c4b22b39e84ecc201f025_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552377021/Put/seqid=0 2024-11-14T02:46:17,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742351_1527 (size=12304) 2024-11-14T02:46:17,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:17,913 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144df6e08b582c4b22b39e84ecc201f025_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144df6e08b582c4b22b39e84ecc201f025_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:17,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/f856eaa9ad494d79b0e3477eaa227576, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:17,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/f856eaa9ad494d79b0e3477eaa227576 is 175, key is test_row_0/A:col10/1731552377021/Put/seqid=0 2024-11-14T02:46:17,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742352_1528 (size=31105) 2024-11-14T02:46:17,966 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/f856eaa9ad494d79b0e3477eaa227576 2024-11-14T02:46:17,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/a15e889e8039463cae2f6cf2adb836b2 is 50, key is test_row_0/B:col10/1731552377021/Put/seqid=0 2024-11-14T02:46:17,993 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/22a01da3a4824ceb84703ccdf3c0653b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/22a01da3a4824ceb84703ccdf3c0653b 2024-11-14T02:46:18,001 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/C of 672d111a762fbc594a3a447bdac7fae8 into 22a01da3a4824ceb84703ccdf3c0653b(size=12.3 K), total size for store is 21.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:18,001 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:18,001 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/C, priority=13, startTime=1731552376956; duration=0sec 2024-11-14T02:46:18,001 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:18,001 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:C 2024-11-14T02:46:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T02:46:18,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742353_1529 (size=12151) 2024-11-14T02:46:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:18,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:18,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552438220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552438224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552438226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552438226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T02:46:18,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552438336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552438336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552438337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552438337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,446 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/a15e889e8039463cae2f6cf2adb836b2 2024-11-14T02:46:18,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/25fe503ef08f4e9087a33d4face8e8b7 is 50, key is test_row_0/C:col10/1731552377021/Put/seqid=0 2024-11-14T02:46:18,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742354_1530 (size=12151) 2024-11-14T02:46:18,470 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/25fe503ef08f4e9087a33d4face8e8b7 2024-11-14T02:46:18,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/f856eaa9ad494d79b0e3477eaa227576 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/f856eaa9ad494d79b0e3477eaa227576 2024-11-14T02:46:18,476 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/f856eaa9ad494d79b0e3477eaa227576, entries=150, sequenceid=237, filesize=30.4 K 2024-11-14T02:46:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/a15e889e8039463cae2f6cf2adb836b2 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a15e889e8039463cae2f6cf2adb836b2 2024-11-14T02:46:18,486 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a15e889e8039463cae2f6cf2adb836b2, entries=150, sequenceid=237, filesize=11.9 K 2024-11-14T02:46:18,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/25fe503ef08f4e9087a33d4face8e8b7 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/25fe503ef08f4e9087a33d4face8e8b7 2024-11-14T02:46:18,491 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/25fe503ef08f4e9087a33d4face8e8b7, entries=150, sequenceid=237, filesize=11.9 K 2024-11-14T02:46:18,491 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 672d111a762fbc594a3a447bdac7fae8 in 650ms, sequenceid=237, compaction requested=true 2024-11-14T02:46:18,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:18,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:18,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-14T02:46:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-14T02:46:18,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-14T02:46:18,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 810 msec 2024-11-14T02:46:18,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 820 msec 2024-11-14T02:46:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:18,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T02:46:18,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:18,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:18,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:18,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:18,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:18,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:18,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114da90bf6c3fa14218baf08e5ddf664445_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552378223/Put/seqid=0 2024-11-14T02:46:18,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742355_1531 (size=12354) 2024-11-14T02:46:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552438568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552438569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552438571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552438573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552438686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552438686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552438686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552438686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T02:46:18,825 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:18,825 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-14T02:46:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T02:46:18,827 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:18,827 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:18,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:18,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552438896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552438896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552438896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552438905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T02:46:18,971 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:18,977 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114da90bf6c3fa14218baf08e5ddf664445_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114da90bf6c3fa14218baf08e5ddf664445_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:18,978 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/55deae69957549b9b6f9e9674c3f30d5, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:18,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/55deae69957549b9b6f9e9674c3f30d5 is 175, key is test_row_0/A:col10/1731552378223/Put/seqid=0 2024-11-14T02:46:18,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T02:46:18,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:18,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:18,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:18,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:18,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:18,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742356_1532 (size=31155) 2024-11-14T02:46:19,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T02:46:19,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:19,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:19,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:19,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T02:46:19,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552439205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:19,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552439205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:19,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552439205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:19,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552439215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:19,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:19,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T02:46:19,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:19,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:19,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:19,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,386 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/55deae69957549b9b6f9e9674c3f30d5 2024-11-14T02:46:19,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/501f873500c544089bacc3491ec42936 is 50, key is test_row_0/B:col10/1731552378223/Put/seqid=0 2024-11-14T02:46:19,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742357_1533 (size=12201) 2024-11-14T02:46:19,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T02:46:19,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:19,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:19,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:19,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T02:46:19,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T02:46:19,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:19,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:19,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:19,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552439715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:19,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552439715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:19,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:19,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552439715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:19,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552439725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:19,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:19,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:19,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T02:46:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:19,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/501f873500c544089bacc3491ec42936 2024-11-14T02:46:19,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/aa7363128ac5497ba92032ffc268e067 is 50, key is test_row_0/C:col10/1731552378223/Put/seqid=0 2024-11-14T02:46:19,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742358_1534 (size=12201) 2024-11-14T02:46:19,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T02:46:19,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:19,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:19,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:19,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:19,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T02:46:20,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T02:46:20,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:20,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:20,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:20,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:20,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:20,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T02:46:20,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:20,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:20,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:20,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:20,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:20,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/aa7363128ac5497ba92032ffc268e067 2024-11-14T02:46:20,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/55deae69957549b9b6f9e9674c3f30d5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/55deae69957549b9b6f9e9674c3f30d5 2024-11-14T02:46:20,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/55deae69957549b9b6f9e9674c3f30d5, entries=150, sequenceid=257, filesize=30.4 K 2024-11-14T02:46:20,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/501f873500c544089bacc3491ec42936 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/501f873500c544089bacc3491ec42936 2024-11-14T02:46:20,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/501f873500c544089bacc3491ec42936, entries=150, sequenceid=257, filesize=11.9 K 2024-11-14T02:46:20,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/aa7363128ac5497ba92032ffc268e067 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/aa7363128ac5497ba92032ffc268e067 2024-11-14T02:46:20,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/aa7363128ac5497ba92032ffc268e067, entries=150, sequenceid=257, filesize=11.9 K 2024-11-14T02:46:20,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 672d111a762fbc594a3a447bdac7fae8 in 1691ms, sequenceid=257, compaction requested=true 2024-11-14T02:46:20,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:20,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:46:20,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:20,241 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:46:20,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:46:20,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:20,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:46:20,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T02:46:20,241 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:46:20,242 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46670 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:46:20,242 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/B is initiating minor compaction (all files) 2024-11-14T02:46:20,243 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/B in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:20,243 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/f42c02b6276442209ec25053993db406, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/9be3e2576ac24e01830a5fe26e42d34c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a15e889e8039463cae2f6cf2adb836b2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/501f873500c544089bacc3491ec42936] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=45.6 K 2024-11-14T02:46:20,243 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124876 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:46:20,243 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/A is initiating minor compaction (all files) 2024-11-14T02:46:20,243 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/A in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:20,243 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/cc094a2f5717472a983572a263d64f38, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/4ea2e70138a8472db755ad99c58c4d14, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/f856eaa9ad494d79b0e3477eaa227576, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/55deae69957549b9b6f9e9674c3f30d5] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=121.9 K 2024-11-14T02:46:20,243 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:20,243 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/cc094a2f5717472a983572a263d64f38, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/4ea2e70138a8472db755ad99c58c4d14, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/f856eaa9ad494d79b0e3477eaa227576, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/55deae69957549b9b6f9e9674c3f30d5] 2024-11-14T02:46:20,244 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting f42c02b6276442209ec25053993db406, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731552375165 2024-11-14T02:46:20,244 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc094a2f5717472a983572a263d64f38, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731552375165 2024-11-14T02:46:20,244 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ea2e70138a8472db755ad99c58c4d14, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731552376338 2024-11-14T02:46:20,244 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 9be3e2576ac24e01830a5fe26e42d34c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731552376338 2024-11-14T02:46:20,245 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting a15e889e8039463cae2f6cf2adb836b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731552377004 2024-11-14T02:46:20,245 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting f856eaa9ad494d79b0e3477eaa227576, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731552377004 2024-11-14T02:46:20,245 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55deae69957549b9b6f9e9674c3f30d5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731552378223 2024-11-14T02:46:20,245 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 501f873500c544089bacc3491ec42936, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731552378223 2024-11-14T02:46:20,256 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#B#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:20,256 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/31f6daab974d48b1b74ad20cf840e0d9 is 50, key is test_row_0/B:col10/1731552378223/Put/seqid=0 2024-11-14T02:46:20,261 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:20,267 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e1db57d31c5242b09c22b17763c50436_672d111a762fbc594a3a447bdac7fae8 store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:20,270 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e1db57d31c5242b09c22b17763c50436_672d111a762fbc594a3a447bdac7fae8, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:20,270 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e1db57d31c5242b09c22b17763c50436_672d111a762fbc594a3a447bdac7fae8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:20,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742359_1535 (size=12747) 2024-11-14T02:46:20,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742360_1536 (size=4469) 2024-11-14T02:46:20,288 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#A#compaction#450 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:20,288 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/8024cb27701b4c4dbdb9ace055751eda is 175, key is test_row_0/A:col10/1731552378223/Put/seqid=0 2024-11-14T02:46:20,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742361_1537 (size=31701) 2024-11-14T02:46:20,296 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/8024cb27701b4c4dbdb9ace055751eda as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/8024cb27701b4c4dbdb9ace055751eda 2024-11-14T02:46:20,300 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/A of 672d111a762fbc594a3a447bdac7fae8 into 8024cb27701b4c4dbdb9ace055751eda(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:20,300 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:20,300 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/A, priority=12, startTime=1731552380241; duration=0sec 2024-11-14T02:46:20,300 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:20,300 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:A 2024-11-14T02:46:20,301 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:46:20,303 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46670 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:46:20,303 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/C is initiating minor compaction (all files) 2024-11-14T02:46:20,303 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/C in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:20,303 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/22a01da3a4824ceb84703ccdf3c0653b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/77b511beecb444289a18e2f1c3981290, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/25fe503ef08f4e9087a33d4face8e8b7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/aa7363128ac5497ba92032ffc268e067] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=45.6 K 2024-11-14T02:46:20,303 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22a01da3a4824ceb84703ccdf3c0653b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731552375165 2024-11-14T02:46:20,303 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77b511beecb444289a18e2f1c3981290, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731552376338 2024-11-14T02:46:20,304 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25fe503ef08f4e9087a33d4face8e8b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731552377004 2024-11-14T02:46:20,304 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa7363128ac5497ba92032ffc268e067, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731552378223 2024-11-14T02:46:20,312 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#C#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:20,313 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/eaf1ceebb79f49fd839be923b25ad790 is 50, key is test_row_0/C:col10/1731552378223/Put/seqid=0 2024-11-14T02:46:20,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742362_1538 (size=12747) 2024-11-14T02:46:20,339 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/eaf1ceebb79f49fd839be923b25ad790 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/eaf1ceebb79f49fd839be923b25ad790 2024-11-14T02:46:20,345 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/C of 672d111a762fbc594a3a447bdac7fae8 into eaf1ceebb79f49fd839be923b25ad790(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:20,346 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:20,346 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/C, priority=12, startTime=1731552380241; duration=0sec 2024-11-14T02:46:20,346 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:20,346 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:C 2024-11-14T02:46:20,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T02:46:20,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:20,353 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T02:46:20,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:20,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:20,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:20,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:20,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:20,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:20,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e4bc35c6854a4ce186e4f67c11be6496_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552378565/Put/seqid=0 2024-11-14T02:46:20,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742363_1539 (size=12454) 2024-11-14T02:46:20,676 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/31f6daab974d48b1b74ad20cf840e0d9 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/31f6daab974d48b1b74ad20cf840e0d9 2024-11-14T02:46:20,680 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/B of 672d111a762fbc594a3a447bdac7fae8 into 31f6daab974d48b1b74ad20cf840e0d9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:20,680 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:20,680 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/B, priority=12, startTime=1731552380241; duration=0sec 2024-11-14T02:46:20,680 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:20,680 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:B 2024-11-14T02:46:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:20,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:20,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552440751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:20,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552440752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:20,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552440752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:20,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552440753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:20,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:20,784 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e4bc35c6854a4ce186e4f67c11be6496_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e4bc35c6854a4ce186e4f67c11be6496_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:20,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/774f4657bed04305bfdae7c91d63169b, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:20,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/774f4657bed04305bfdae7c91d63169b is 175, key is test_row_0/A:col10/1731552378565/Put/seqid=0 2024-11-14T02:46:20,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742364_1540 (size=31255) 2024-11-14T02:46:20,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552440865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552440865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:20,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552440865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552440866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T02:46:21,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552441075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552441075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552441076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552441076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,190 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/774f4657bed04305bfdae7c91d63169b 2024-11-14T02:46:21,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/b2402c4aff38443e95783b13dce079ea is 50, key is test_row_0/B:col10/1731552378565/Put/seqid=0 2024-11-14T02:46:21,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742365_1541 (size=12301) 2024-11-14T02:46:21,200 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/b2402c4aff38443e95783b13dce079ea 2024-11-14T02:46:21,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/77adc067e11545c3ad4bddf1df6d462a is 50, key is test_row_0/C:col10/1731552378565/Put/seqid=0 2024-11-14T02:46:21,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742366_1542 (size=12301) 2024-11-14T02:46:21,211 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/77adc067e11545c3ad4bddf1df6d462a 2024-11-14T02:46:21,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/774f4657bed04305bfdae7c91d63169b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/774f4657bed04305bfdae7c91d63169b 2024-11-14T02:46:21,221 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/774f4657bed04305bfdae7c91d63169b, entries=150, sequenceid=272, filesize=30.5 K 2024-11-14T02:46:21,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/b2402c4aff38443e95783b13dce079ea as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b2402c4aff38443e95783b13dce079ea 2024-11-14T02:46:21,225 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b2402c4aff38443e95783b13dce079ea, entries=150, sequenceid=272, filesize=12.0 K 2024-11-14T02:46:21,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/77adc067e11545c3ad4bddf1df6d462a as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/77adc067e11545c3ad4bddf1df6d462a 2024-11-14T02:46:21,229 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/77adc067e11545c3ad4bddf1df6d462a, entries=150, sequenceid=272, filesize=12.0 K 2024-11-14T02:46:21,230 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 672d111a762fbc594a3a447bdac7fae8 in 876ms, sequenceid=272, compaction requested=false 2024-11-14T02:46:21,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:21,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:21,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-14T02:46:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-14T02:46:21,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-14T02:46:21,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4050 sec 2024-11-14T02:46:21,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.4140 sec 2024-11-14T02:46:21,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:21,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T02:46:21,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:21,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:46:21,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:21,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:46:21,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:21,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T02:46:21,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e51a60cc9a0948498bf7626b2bc9d8f5_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552381386/Put/seqid=0 2024-11-14T02:46:21,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742367_1543 (size=14994) 2024-11-14T02:46:21,396 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:21,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552441393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552441394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,399 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e51a60cc9a0948498bf7626b2bc9d8f5_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e51a60cc9a0948498bf7626b2bc9d8f5_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552441395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,400 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/c86bcab4d8794d48abbe5149ba9d5d6d, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:21,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/c86bcab4d8794d48abbe5149ba9d5d6d is 175, key is test_row_0/A:col10/1731552381386/Put/seqid=0 2024-11-14T02:46:21,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552441396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742368_1544 (size=39949) 2024-11-14T02:46:21,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552441505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552441505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552441505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552441505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552441715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552441715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552441716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552441715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:21,804 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/c86bcab4d8794d48abbe5149ba9d5d6d 2024-11-14T02:46:21,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/b182cfc037ee4de7af1a21c85152f4e7 is 50, key is test_row_0/B:col10/1731552381386/Put/seqid=0 2024-11-14T02:46:21,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742369_1545 (size=12301) 2024-11-14T02:46:21,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/b182cfc037ee4de7af1a21c85152f4e7 2024-11-14T02:46:21,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/4c561d4b3c184c3a901125074bdb5858 is 50, key is test_row_0/C:col10/1731552381386/Put/seqid=0 2024-11-14T02:46:21,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742370_1546 (size=12301) 2024-11-14T02:46:22,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552442026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552442026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552442026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552442026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/4c561d4b3c184c3a901125074bdb5858 2024-11-14T02:46:22,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/c86bcab4d8794d48abbe5149ba9d5d6d as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/c86bcab4d8794d48abbe5149ba9d5d6d 2024-11-14T02:46:22,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/c86bcab4d8794d48abbe5149ba9d5d6d, entries=200, sequenceid=294, filesize=39.0 K 2024-11-14T02:46:22,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/b182cfc037ee4de7af1a21c85152f4e7 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b182cfc037ee4de7af1a21c85152f4e7 2024-11-14T02:46:22,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b182cfc037ee4de7af1a21c85152f4e7, entries=150, sequenceid=294, filesize=12.0 K 2024-11-14T02:46:22,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/4c561d4b3c184c3a901125074bdb5858 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/4c561d4b3c184c3a901125074bdb5858 2024-11-14T02:46:22,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/4c561d4b3c184c3a901125074bdb5858, entries=150, sequenceid=294, filesize=12.0 K 2024-11-14T02:46:22,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 672d111a762fbc594a3a447bdac7fae8 in 858ms, sequenceid=294, compaction requested=true 2024-11-14T02:46:22,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:22,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:46:22,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:22,246 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:22,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:46:22,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:22,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:46:22,246 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:22,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:22,247 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37349 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:22,247 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102905 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:22,248 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/B is initiating minor compaction (all files) 2024-11-14T02:46:22,248 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/A is initiating minor compaction (all files) 2024-11-14T02:46:22,248 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/B in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:22,248 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/A in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:22,248 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/31f6daab974d48b1b74ad20cf840e0d9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b2402c4aff38443e95783b13dce079ea, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b182cfc037ee4de7af1a21c85152f4e7] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=36.5 K 2024-11-14T02:46:22,248 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/8024cb27701b4c4dbdb9ace055751eda, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/774f4657bed04305bfdae7c91d63169b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/c86bcab4d8794d48abbe5149ba9d5d6d] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=100.5 K 2024-11-14T02:46:22,248 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:22,248 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/8024cb27701b4c4dbdb9ace055751eda, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/774f4657bed04305bfdae7c91d63169b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/c86bcab4d8794d48abbe5149ba9d5d6d] 2024-11-14T02:46:22,248 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 31f6daab974d48b1b74ad20cf840e0d9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731552378223 2024-11-14T02:46:22,248 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8024cb27701b4c4dbdb9ace055751eda, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731552378223 2024-11-14T02:46:22,248 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting b2402c4aff38443e95783b13dce079ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731552378565 2024-11-14T02:46:22,248 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 774f4657bed04305bfdae7c91d63169b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731552378565 2024-11-14T02:46:22,248 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting b182cfc037ee4de7af1a21c85152f4e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731552380732 2024-11-14T02:46:22,249 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting c86bcab4d8794d48abbe5149ba9d5d6d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731552380732 2024-11-14T02:46:22,257 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:22,258 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#B#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:22,259 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/0d125ea5203b4b258c274362e01671d6 is 50, key is test_row_0/B:col10/1731552381386/Put/seqid=0 2024-11-14T02:46:22,273 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e41daa06f0ca4cbc889ef7ca8a8ae2a7_672d111a762fbc594a3a447bdac7fae8 store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:22,275 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e41daa06f0ca4cbc889ef7ca8a8ae2a7_672d111a762fbc594a3a447bdac7fae8, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:22,275 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e41daa06f0ca4cbc889ef7ca8a8ae2a7_672d111a762fbc594a3a447bdac7fae8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:22,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742371_1547 (size=12949) 2024-11-14T02:46:22,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742372_1548 (size=4469) 2024-11-14T02:46:22,279 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#A#compaction#459 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:22,280 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/64becffcfb4e4de3b5bf91c1ca16dde0 is 175, key is test_row_0/A:col10/1731552381386/Put/seqid=0 2024-11-14T02:46:22,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742373_1549 (size=31903) 2024-11-14T02:46:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:22,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T02:46:22,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:22,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:22,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:22,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:22,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:22,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:22,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111402cae0d24c7541b1bd34dea8722752dc_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552382536/Put/seqid=0 2024-11-14T02:46:22,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742374_1550 (size=14994) 2024-11-14T02:46:22,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552442587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552442591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552442592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552442592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,680 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/0d125ea5203b4b258c274362e01671d6 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/0d125ea5203b4b258c274362e01671d6 2024-11-14T02:46:22,691 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/B of 672d111a762fbc594a3a447bdac7fae8 into 0d125ea5203b4b258c274362e01671d6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:22,691 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:22,691 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/B, priority=13, startTime=1731552382246; duration=0sec 2024-11-14T02:46:22,691 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:22,691 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:B 2024-11-14T02:46:22,691 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:22,692 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/64becffcfb4e4de3b5bf91c1ca16dde0 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/64becffcfb4e4de3b5bf91c1ca16dde0 2024-11-14T02:46:22,692 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37349 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:22,692 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/C is initiating minor compaction (all files) 2024-11-14T02:46:22,692 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/C in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:22,692 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/eaf1ceebb79f49fd839be923b25ad790, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/77adc067e11545c3ad4bddf1df6d462a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/4c561d4b3c184c3a901125074bdb5858] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=36.5 K 2024-11-14T02:46:22,693 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting eaf1ceebb79f49fd839be923b25ad790, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731552378223 2024-11-14T02:46:22,693 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 77adc067e11545c3ad4bddf1df6d462a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731552378565 2024-11-14T02:46:22,693 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c561d4b3c184c3a901125074bdb5858, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731552380732 2024-11-14T02:46:22,697 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/A of 672d111a762fbc594a3a447bdac7fae8 into 64becffcfb4e4de3b5bf91c1ca16dde0(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:22,697 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:22,697 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/A, priority=13, startTime=1731552382246; duration=0sec 2024-11-14T02:46:22,698 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:22,698 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:A 2024-11-14T02:46:22,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552442695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,700 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:22,700 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/8691d59a544b44fabc27bd1f15d97fc2 is 50, key is test_row_0/C:col10/1731552381386/Put/seqid=0 2024-11-14T02:46:22,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742375_1551 (size=12949) 2024-11-14T02:46:22,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552442705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552442705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552442705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552442905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552442916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552442916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552442916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:22,948 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:22,951 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111402cae0d24c7541b1bd34dea8722752dc_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111402cae0d24c7541b1bd34dea8722752dc_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:22,952 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/edf75cb2a5fb4eafaf1c4f6d4867f9a8, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:22,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/edf75cb2a5fb4eafaf1c4f6d4867f9a8 is 175, key is test_row_0/A:col10/1731552382536/Put/seqid=0 2024-11-14T02:46:22,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742376_1552 (size=39949) 2024-11-14T02:46:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T02:46:22,986 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:22,986 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-14T02:46:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T02:46:22,988 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:22,989 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:22,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T02:46:23,112 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/8691d59a544b44fabc27bd1f15d97fc2 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/8691d59a544b44fabc27bd1f15d97fc2 2024-11-14T02:46:23,116 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/C of 672d111a762fbc594a3a447bdac7fae8 into 8691d59a544b44fabc27bd1f15d97fc2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:23,116 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:23,116 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/C, priority=13, startTime=1731552382246; duration=0sec 2024-11-14T02:46:23,116 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:23,116 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:C 2024-11-14T02:46:23,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T02:46:23,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:23,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:23,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:23,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:23,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:23,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552443216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:23,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:23,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552443225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:23,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552443225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:23,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552443225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:23,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T02:46:23,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:23,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:23,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:23,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:23,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T02:46:23,358 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/edf75cb2a5fb4eafaf1c4f6d4867f9a8 2024-11-14T02:46:23,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/552e599e78f7442bb8ad6d958f07aaa3 is 50, key is test_row_0/B:col10/1731552382536/Put/seqid=0 2024-11-14T02:46:23,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742377_1553 (size=12301) 2024-11-14T02:46:23,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T02:46:23,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:23,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:23,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:23,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:23,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:23,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T02:46:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:23,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T02:46:23,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552443725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:23,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552443735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:23,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552443735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552443735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:23,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T02:46:23,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:23,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:23,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:23,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:23,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:23,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/552e599e78f7442bb8ad6d958f07aaa3 2024-11-14T02:46:23,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/ab3aeac06b6d4366bab9792d991acbb6 is 50, key is test_row_0/C:col10/1731552382536/Put/seqid=0 2024-11-14T02:46:23,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742378_1554 (size=12301) 2024-11-14T02:46:23,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/ab3aeac06b6d4366bab9792d991acbb6 2024-11-14T02:46:23,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/edf75cb2a5fb4eafaf1c4f6d4867f9a8 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/edf75cb2a5fb4eafaf1c4f6d4867f9a8 2024-11-14T02:46:23,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/edf75cb2a5fb4eafaf1c4f6d4867f9a8, entries=200, sequenceid=306, filesize=39.0 K 2024-11-14T02:46:23,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/552e599e78f7442bb8ad6d958f07aaa3 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/552e599e78f7442bb8ad6d958f07aaa3 2024-11-14T02:46:23,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/552e599e78f7442bb8ad6d958f07aaa3, entries=150, sequenceid=306, filesize=12.0 K 2024-11-14T02:46:23,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/ab3aeac06b6d4366bab9792d991acbb6 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/ab3aeac06b6d4366bab9792d991acbb6 2024-11-14T02:46:23,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/ab3aeac06b6d4366bab9792d991acbb6, entries=150, sequenceid=306, filesize=12.0 K 2024-11-14T02:46:23,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 672d111a762fbc594a3a447bdac7fae8 in 1270ms, sequenceid=306, compaction requested=false 2024-11-14T02:46:23,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:23,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T02:46:23,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:23,905 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T02:46:23,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:23,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:23,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:23,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:23,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:23,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:23,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141f7299b58253492bb19117a1871b7d86_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552382590/Put/seqid=0 2024-11-14T02:46:23,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742379_1555 (size=12454) 2024-11-14T02:46:23,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:23,918 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141f7299b58253492bb19117a1871b7d86_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141f7299b58253492bb19117a1871b7d86_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:23,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/c676b08974084483a426945ee4dfcb95, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:23,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/c676b08974084483a426945ee4dfcb95 is 175, key is test_row_0/A:col10/1731552382590/Put/seqid=0 2024-11-14T02:46:23,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742380_1556 (size=31255) 2024-11-14T02:46:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T02:46:24,329 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/c676b08974084483a426945ee4dfcb95 2024-11-14T02:46:24,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/a57b3c927c6c4e63bc4f4c0aacb231b9 is 50, key is test_row_0/B:col10/1731552382590/Put/seqid=0 2024-11-14T02:46:24,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742381_1557 (size=12301) 2024-11-14T02:46:24,741 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/a57b3c927c6c4e63bc4f4c0aacb231b9 2024-11-14T02:46:24,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/645a74eece014841a907ed1ba657cf08 is 50, key is test_row_0/C:col10/1731552382590/Put/seqid=0 2024-11-14T02:46:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:24,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:24,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552444758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:24,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552444759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:24,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552444762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:24,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552444763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:24,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742382_1558 (size=12301) 2024-11-14T02:46:24,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552444875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:24,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552444875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:24,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552444875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552444875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552445085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552445085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552445085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552445085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T02:46:25,171 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/645a74eece014841a907ed1ba657cf08 2024-11-14T02:46:25,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/c676b08974084483a426945ee4dfcb95 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/c676b08974084483a426945ee4dfcb95 2024-11-14T02:46:25,180 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/c676b08974084483a426945ee4dfcb95, entries=150, sequenceid=329, filesize=30.5 K 2024-11-14T02:46:25,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/a57b3c927c6c4e63bc4f4c0aacb231b9 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a57b3c927c6c4e63bc4f4c0aacb231b9 2024-11-14T02:46:25,186 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a57b3c927c6c4e63bc4f4c0aacb231b9, entries=150, sequenceid=329, filesize=12.0 K 2024-11-14T02:46:25,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/645a74eece014841a907ed1ba657cf08 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/645a74eece014841a907ed1ba657cf08 2024-11-14T02:46:25,190 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/645a74eece014841a907ed1ba657cf08, entries=150, sequenceid=329, filesize=12.0 K 2024-11-14T02:46:25,191 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 672d111a762fbc594a3a447bdac7fae8 in 1286ms, sequenceid=329, compaction requested=true 2024-11-14T02:46:25,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:25,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:25,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-14T02:46:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-14T02:46:25,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-14T02:46:25,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2040 sec 2024-11-14T02:46:25,196 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T02:46:25,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.2090 sec 2024-11-14T02:46:25,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:25,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T02:46:25,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:25,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:25,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:25,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:25,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:25,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:25,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f5d9fa7141324c73bcb432c6cabee225_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552384762/Put/seqid=0 2024-11-14T02:46:25,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552445423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552445424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742383_1559 (size=12454) 2024-11-14T02:46:25,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552445429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552445429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552445535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552445535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552445545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552445545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552445745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552445745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552445755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552445755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:25,834 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:25,837 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f5d9fa7141324c73bcb432c6cabee225_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f5d9fa7141324c73bcb432c6cabee225_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:25,837 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/7c31807ed1b44766b58b5760d4207b3e, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:25,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/7c31807ed1b44766b58b5760d4207b3e is 175, key is test_row_0/A:col10/1731552384762/Put/seqid=0 2024-11-14T02:46:25,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742384_1560 (size=31255) 2024-11-14T02:46:26,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552446055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:26,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552446055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:26,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552446065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:26,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552446065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:26,241 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/7c31807ed1b44766b58b5760d4207b3e 2024-11-14T02:46:26,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/25e8f14b2fef49acbc7b04358337c810 is 50, key is test_row_0/B:col10/1731552384762/Put/seqid=0 2024-11-14T02:46:26,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742385_1561 (size=12301) 2024-11-14T02:46:26,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552446565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:26,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552446565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:26,572 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:26,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552446575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:26,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552446585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:26,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:26,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:26,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/25e8f14b2fef49acbc7b04358337c810 2024-11-14T02:46:26,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/4910fd005b834f209c74c656e13a8454 is 50, key is test_row_0/C:col10/1731552384762/Put/seqid=0 2024-11-14T02:46:26,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742386_1562 (size=12301) 2024-11-14T02:46:26,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/4910fd005b834f209c74c656e13a8454 2024-11-14T02:46:26,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/7c31807ed1b44766b58b5760d4207b3e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/7c31807ed1b44766b58b5760d4207b3e 2024-11-14T02:46:26,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/7c31807ed1b44766b58b5760d4207b3e, entries=150, sequenceid=342, filesize=30.5 K 2024-11-14T02:46:26,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/25e8f14b2fef49acbc7b04358337c810 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/25e8f14b2fef49acbc7b04358337c810 2024-11-14T02:46:26,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/25e8f14b2fef49acbc7b04358337c810, entries=150, sequenceid=342, filesize=12.0 K 2024-11-14T02:46:26,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/4910fd005b834f209c74c656e13a8454 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/4910fd005b834f209c74c656e13a8454 2024-11-14T02:46:26,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/4910fd005b834f209c74c656e13a8454, entries=150, sequenceid=342, filesize=12.0 K 2024-11-14T02:46:26,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 672d111a762fbc594a3a447bdac7fae8 in 1306ms, sequenceid=342, compaction requested=true 2024-11-14T02:46:26,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:26,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:46:26,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:26,703 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:46:26,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:46:26,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:26,703 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:46:26,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:46:26,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:26,704 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:46:26,704 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:46:26,704 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/A is initiating minor compaction (all files) 2024-11-14T02:46:26,704 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/B is initiating minor compaction (all files) 2024-11-14T02:46:26,704 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/B in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:26,704 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/A in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:26,705 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/0d125ea5203b4b258c274362e01671d6, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/552e599e78f7442bb8ad6d958f07aaa3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a57b3c927c6c4e63bc4f4c0aacb231b9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/25e8f14b2fef49acbc7b04358337c810] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=48.7 K 2024-11-14T02:46:26,705 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/64becffcfb4e4de3b5bf91c1ca16dde0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/edf75cb2a5fb4eafaf1c4f6d4867f9a8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/c676b08974084483a426945ee4dfcb95, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/7c31807ed1b44766b58b5760d4207b3e] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=131.2 K 2024-11-14T02:46:26,705 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:26,705 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/64becffcfb4e4de3b5bf91c1ca16dde0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/edf75cb2a5fb4eafaf1c4f6d4867f9a8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/c676b08974084483a426945ee4dfcb95, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/7c31807ed1b44766b58b5760d4207b3e] 2024-11-14T02:46:26,705 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64becffcfb4e4de3b5bf91c1ca16dde0, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731552380732 2024-11-14T02:46:26,705 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d125ea5203b4b258c274362e01671d6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731552380732 2024-11-14T02:46:26,705 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 552e599e78f7442bb8ad6d958f07aaa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731552381393 2024-11-14T02:46:26,705 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting edf75cb2a5fb4eafaf1c4f6d4867f9a8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731552381389 2024-11-14T02:46:26,706 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting a57b3c927c6c4e63bc4f4c0aacb231b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731552382590 2024-11-14T02:46:26,706 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting c676b08974084483a426945ee4dfcb95, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731552382590 2024-11-14T02:46:26,706 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 25e8f14b2fef49acbc7b04358337c810, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731552384761 2024-11-14T02:46:26,706 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c31807ed1b44766b58b5760d4207b3e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731552384761 2024-11-14T02:46:26,713 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#B#compaction#470 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:26,713 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/95370410e55a4460960e968ad19c4e7b is 50, key is test_row_0/B:col10/1731552384762/Put/seqid=0 2024-11-14T02:46:26,716 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:26,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742387_1563 (size=13085) 2024-11-14T02:46:26,721 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411149e7d7e13ba874ba49ca66c00d98638f7_672d111a762fbc594a3a447bdac7fae8 store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:26,723 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411149e7d7e13ba874ba49ca66c00d98638f7_672d111a762fbc594a3a447bdac7fae8, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:26,723 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149e7d7e13ba874ba49ca66c00d98638f7_672d111a762fbc594a3a447bdac7fae8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:26,726 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/95370410e55a4460960e968ad19c4e7b as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/95370410e55a4460960e968ad19c4e7b 2024-11-14T02:46:26,731 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/B of 672d111a762fbc594a3a447bdac7fae8 into 95370410e55a4460960e968ad19c4e7b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:26,731 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:26,732 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/B, priority=12, startTime=1731552386703; duration=0sec 2024-11-14T02:46:26,732 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:26,732 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:B 2024-11-14T02:46:26,732 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T02:46:26,733 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T02:46:26,733 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/C is initiating minor compaction (all files) 2024-11-14T02:46:26,733 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/C in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:26,734 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/8691d59a544b44fabc27bd1f15d97fc2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/ab3aeac06b6d4366bab9792d991acbb6, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/645a74eece014841a907ed1ba657cf08, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/4910fd005b834f209c74c656e13a8454] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=48.7 K 2024-11-14T02:46:26,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742388_1564 (size=4469) 2024-11-14T02:46:26,735 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 8691d59a544b44fabc27bd1f15d97fc2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731552380732 2024-11-14T02:46:26,736 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting ab3aeac06b6d4366bab9792d991acbb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731552381393 2024-11-14T02:46:26,736 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#A#compaction#471 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:26,736 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 645a74eece014841a907ed1ba657cf08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731552382590 2024-11-14T02:46:26,736 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 4910fd005b834f209c74c656e13a8454, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731552384761 2024-11-14T02:46:26,736 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/b78a5325b88248678989471cff5250b1 is 175, key is test_row_0/A:col10/1731552384762/Put/seqid=0 2024-11-14T02:46:26,767 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#C#compaction#472 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:26,767 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/07a532a4bb784bffb7c5cf4a396fc761 is 50, key is test_row_0/C:col10/1731552384762/Put/seqid=0 2024-11-14T02:46:26,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742389_1565 (size=32039) 2024-11-14T02:46:26,788 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/b78a5325b88248678989471cff5250b1 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/b78a5325b88248678989471cff5250b1 2024-11-14T02:46:26,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742390_1566 (size=13085) 2024-11-14T02:46:26,796 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/A of 672d111a762fbc594a3a447bdac7fae8 into b78a5325b88248678989471cff5250b1(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:26,796 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:26,796 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/A, priority=12, startTime=1731552386703; duration=0sec 2024-11-14T02:46:26,796 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:26,796 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:A 2024-11-14T02:46:26,800 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/07a532a4bb784bffb7c5cf4a396fc761 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/07a532a4bb784bffb7c5cf4a396fc761 2024-11-14T02:46:26,806 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/C of 672d111a762fbc594a3a447bdac7fae8 into 07a532a4bb784bffb7c5cf4a396fc761(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:26,806 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:26,806 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/C, priority=12, startTime=1731552386703; duration=0sec 2024-11-14T02:46:26,806 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:26,807 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:C 2024-11-14T02:46:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T02:46:27,146 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:27,146 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-14T02:46:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T02:46:27,148 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:27,149 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:27,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T02:46:27,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T02:46:27,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:27,302 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T02:46:27,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:27,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:27,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:27,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:27,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:27,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:27,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111402188353d7734e7cb6b3ac1020b45e32_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552385427/Put/seqid=0 2024-11-14T02:46:27,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742391_1567 (size=12454) 2024-11-14T02:46:27,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:27,314 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111402188353d7734e7cb6b3ac1020b45e32_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111402188353d7734e7cb6b3ac1020b45e32_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:27,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/05d9c5bcda1c4cb089b28d631189c78a, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:27,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/05d9c5bcda1c4cb089b28d631189c78a is 175, key is test_row_0/A:col10/1731552385427/Put/seqid=0 2024-11-14T02:46:27,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742392_1568 (size=31255) 2024-11-14T02:46:27,322 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/05d9c5bcda1c4cb089b28d631189c78a 2024-11-14T02:46:27,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/ad29154bf94d4b6497f0199c2027e48d is 50, key is test_row_0/B:col10/1731552385427/Put/seqid=0 2024-11-14T02:46:27,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742393_1569 (size=12301) 2024-11-14T02:46:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T02:46:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:27,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:27,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552447510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:27,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57022 deadline: 1731552447575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:27,578 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_2' in 672d111a762fbc594a3a447bdac7fae8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:46:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:27,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57030 deadline: 1731552447586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:27,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57046 deadline: 1731552447586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:27,587 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_2' in 672d111a762fbc594a3a447bdac7fae8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:46:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:27,588 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_0' in 672d111a762fbc594a3a447bdac7fae8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:46:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:27,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56974 deadline: 1731552447605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:27,608 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e5dc8b5cf5e9,46387,1731552206828 for 'test_row_0' in 672d111a762fbc594a3a447bdac7fae8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T02:46:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:27,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:27,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:27,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552447625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:27,732 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/ad29154bf94d4b6497f0199c2027e48d 2024-11-14T02:46:27,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/0ff53a5d02314365bc9cea7b3d7746a9 is 50, key is test_row_0/C:col10/1731552385427/Put/seqid=0 2024-11-14T02:46:27,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742394_1570 (size=12301) 2024-11-14T02:46:27,744 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/0ff53a5d02314365bc9cea7b3d7746a9 2024-11-14T02:46:27,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/05d9c5bcda1c4cb089b28d631189c78a as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/05d9c5bcda1c4cb089b28d631189c78a 2024-11-14T02:46:27,751 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/05d9c5bcda1c4cb089b28d631189c78a, entries=150, sequenceid=365, filesize=30.5 K 2024-11-14T02:46:27,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/ad29154bf94d4b6497f0199c2027e48d as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/ad29154bf94d4b6497f0199c2027e48d 2024-11-14T02:46:27,755 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/ad29154bf94d4b6497f0199c2027e48d, entries=150, sequenceid=365, filesize=12.0 K 2024-11-14T02:46:27,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/0ff53a5d02314365bc9cea7b3d7746a9 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/0ff53a5d02314365bc9cea7b3d7746a9 2024-11-14T02:46:27,760 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/0ff53a5d02314365bc9cea7b3d7746a9, entries=150, sequenceid=365, filesize=12.0 K 2024-11-14T02:46:27,761 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 672d111a762fbc594a3a447bdac7fae8 in 460ms, sequenceid=365, compaction requested=false 2024-11-14T02:46:27,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:27,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:27,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-14T02:46:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-14T02:46:27,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-14T02:46:27,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 615 msec 2024-11-14T02:46:27,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 620 msec 2024-11-14T02:46:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T02:46:27,776 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:27,776 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T02:46:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-14T02:46:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T02:46:27,778 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T02:46:27,779 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T02:46:27,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T02:46:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:27,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T02:46:27,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:27,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:27,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:27,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:27,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:27,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:27,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c3531bfbfd1d4cf6989d1b0adedb0cbe_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552387509/Put/seqid=0 2024-11-14T02:46:27,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742395_1571 (size=12454) 2024-11-14T02:46:27,867 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:27,869 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c3531bfbfd1d4cf6989d1b0adedb0cbe_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c3531bfbfd1d4cf6989d1b0adedb0cbe_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:27,872 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/26ba2827fd244af59548f5dc86606065, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:27,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/26ba2827fd244af59548f5dc86606065 is 175, key is test_row_0/A:col10/1731552387509/Put/seqid=0 2024-11-14T02:46:27,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742396_1572 (size=31255) 2024-11-14T02:46:27,878 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/26ba2827fd244af59548f5dc86606065 2024-11-14T02:46:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T02:46:27,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/b6f84a6fd0e54bc3a2e987f4b48cfd9e is 50, key is test_row_0/B:col10/1731552387509/Put/seqid=0 2024-11-14T02:46:27,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742397_1573 (size=12301) 2024-11-14T02:46:27,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/b6f84a6fd0e54bc3a2e987f4b48cfd9e 2024-11-14T02:46:27,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/2859441cd2c743a1b23ccceedec17b11 is 50, key is test_row_0/C:col10/1731552387509/Put/seqid=0 2024-11-14T02:46:27,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T02:46:27,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:27,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:27,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:27,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:27,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:27,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742398_1574 (size=12301) 2024-11-14T02:46:27,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552447933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:27,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/2859441cd2c743a1b23ccceedec17b11 2024-11-14T02:46:27,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/26ba2827fd244af59548f5dc86606065 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/26ba2827fd244af59548f5dc86606065 2024-11-14T02:46:27,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/26ba2827fd244af59548f5dc86606065, entries=150, sequenceid=378, filesize=30.5 K 2024-11-14T02:46:27,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/b6f84a6fd0e54bc3a2e987f4b48cfd9e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b6f84a6fd0e54bc3a2e987f4b48cfd9e 2024-11-14T02:46:27,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b6f84a6fd0e54bc3a2e987f4b48cfd9e, entries=150, sequenceid=378, filesize=12.0 K 2024-11-14T02:46:27,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/2859441cd2c743a1b23ccceedec17b11 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/2859441cd2c743a1b23ccceedec17b11 2024-11-14T02:46:27,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/2859441cd2c743a1b23ccceedec17b11, entries=150, sequenceid=378, filesize=12.0 K 2024-11-14T02:46:27,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 672d111a762fbc594a3a447bdac7fae8 in 120ms, sequenceid=378, compaction requested=true 2024-11-14T02:46:27,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:27,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T02:46:27,958 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:27,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:27,958 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:27,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T02:46:27,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:27,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 672d111a762fbc594a3a447bdac7fae8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T02:46:27,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:27,959 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:27,959 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/B is initiating minor compaction (all files) 2024-11-14T02:46:27,959 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/B in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:27,959 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/95370410e55a4460960e968ad19c4e7b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/ad29154bf94d4b6497f0199c2027e48d, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b6f84a6fd0e54bc3a2e987f4b48cfd9e] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=36.8 K 2024-11-14T02:46:27,959 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:27,959 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/A is initiating minor compaction (all files) 2024-11-14T02:46:27,959 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/A in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:27,959 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting 95370410e55a4460960e968ad19c4e7b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731552384761 2024-11-14T02:46:27,959 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/b78a5325b88248678989471cff5250b1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/05d9c5bcda1c4cb089b28d631189c78a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/26ba2827fd244af59548f5dc86606065] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=92.3 K 2024-11-14T02:46:27,960 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:27,960 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. files: [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/b78a5325b88248678989471cff5250b1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/05d9c5bcda1c4cb089b28d631189c78a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/26ba2827fd244af59548f5dc86606065] 2024-11-14T02:46:27,960 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting b78a5325b88248678989471cff5250b1, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731552384761 2024-11-14T02:46:27,960 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting ad29154bf94d4b6497f0199c2027e48d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731552385415 2024-11-14T02:46:27,960 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05d9c5bcda1c4cb089b28d631189c78a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731552385415 2024-11-14T02:46:27,960 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] compactions.Compactor(225): Compacting b6f84a6fd0e54bc3a2e987f4b48cfd9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731552387504 2024-11-14T02:46:27,961 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26ba2827fd244af59548f5dc86606065, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731552387504 2024-11-14T02:46:27,969 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#B#compaction#479 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:27,969 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/1e7528b3b3e84eb4b3a56a4c1d8ac841 is 50, key is test_row_0/B:col10/1731552387509/Put/seqid=0 2024-11-14T02:46:27,970 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:27,985 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411146fae667edfd84779a558d9d135fc2070_672d111a762fbc594a3a447bdac7fae8 store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:27,986 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411146fae667edfd84779a558d9d135fc2070_672d111a762fbc594a3a447bdac7fae8, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:27,986 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146fae667edfd84779a558d9d135fc2070_672d111a762fbc594a3a447bdac7fae8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:28,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742399_1575 (size=13187) 2024-11-14T02:46:28,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742400_1576 (size=4469) 2024-11-14T02:46:28,012 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#A#compaction#480 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:28,012 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/161706cac46142c4ac74b4e77d8ed074 is 175, key is test_row_0/A:col10/1731552387509/Put/seqid=0 2024-11-14T02:46:28,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742401_1577 (size=32141) 2024-11-14T02:46:28,031 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/161706cac46142c4ac74b4e77d8ed074 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/161706cac46142c4ac74b4e77d8ed074 2024-11-14T02:46:28,039 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/A of 672d111a762fbc594a3a447bdac7fae8 into 161706cac46142c4ac74b4e77d8ed074(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:28,039 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:28,039 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/A, priority=13, startTime=1731552387958; duration=0sec 2024-11-14T02:46:28,039 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T02:46:28,039 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:A 2024-11-14T02:46:28,039 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T02:46:28,040 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T02:46:28,040 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1541): 672d111a762fbc594a3a447bdac7fae8/C is initiating minor compaction (all files) 2024-11-14T02:46:28,040 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 672d111a762fbc594a3a447bdac7fae8/C in TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:28,040 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/07a532a4bb784bffb7c5cf4a396fc761, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/0ff53a5d02314365bc9cea7b3d7746a9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/2859441cd2c743a1b23ccceedec17b11] into tmpdir=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp, totalSize=36.8 K 2024-11-14T02:46:28,041 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07a532a4bb784bffb7c5cf4a396fc761, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731552384761 2024-11-14T02:46:28,041 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ff53a5d02314365bc9cea7b3d7746a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731552385415 2024-11-14T02:46:28,041 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2859441cd2c743a1b23ccceedec17b11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731552387504 2024-11-14T02:46:28,047 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 672d111a762fbc594a3a447bdac7fae8#C#compaction#481 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T02:46:28,048 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/5bf69f3d90324e95ae87a7a4e074fa92 is 50, key is test_row_0/C:col10/1731552387509/Put/seqid=0 2024-11-14T02:46:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:28,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T02:46:28,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:28,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:28,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:28,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:28,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:28,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:28,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b1a63357e2414c619f012bb20e2b799e_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552388047/Put/seqid=0 2024-11-14T02:46:28,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742402_1578 (size=13187) 2024-11-14T02:46:28,069 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/5bf69f3d90324e95ae87a7a4e074fa92 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/5bf69f3d90324e95ae87a7a4e074fa92 2024-11-14T02:46:28,073 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/C of 672d111a762fbc594a3a447bdac7fae8 into 5bf69f3d90324e95ae87a7a4e074fa92(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:28,073 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:28,073 INFO [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/C, priority=13, startTime=1731552387958; duration=0sec 2024-11-14T02:46:28,073 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:28,073 DEBUG [RS:0;e5dc8b5cf5e9:46387-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:C 2024-11-14T02:46:28,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742403_1579 (size=14994) 2024-11-14T02:46:28,082 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:28,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T02:46:28,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:28,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:28,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:28,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,086 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b1a63357e2414c619f012bb20e2b799e_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b1a63357e2414c619f012bb20e2b799e_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,087 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/7d6dc6c7d77d4aba9a5e3c2327a68963, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:28,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/7d6dc6c7d77d4aba9a5e3c2327a68963 is 175, key is test_row_0/A:col10/1731552388047/Put/seqid=0 2024-11-14T02:46:28,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742404_1580 (size=39949) 2024-11-14T02:46:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T02:46:28,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552448097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:28,202 INFO [Thread-2151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2151. 2024-11-14T02:46:28,202 DEBUG [Thread-2151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:28,202 DEBUG [Thread-2151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:28,202 DEBUG [Thread-2151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:28,202 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:28,203 INFO [Thread-2149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2149. 2024-11-14T02:46:28,203 DEBUG [Thread-2149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:28,203 DEBUG [Thread-2149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:28,203 DEBUG [Thread-2149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:28,203 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:28,205 INFO [Thread-2145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2145. 2024-11-14T02:46:28,205 DEBUG [Thread-2145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:28,205 DEBUG [Thread-2145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:28,205 DEBUG [Thread-2145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:28,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:28,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552448205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:28,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:28,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:28,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:28,206 INFO [Thread-2147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2147. 2024-11-14T02:46:28,206 DEBUG [Thread-2147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:28,206 DEBUG [Thread-2147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:28,206 DEBUG [Thread-2147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:28,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:28,207 INFO [Thread-2143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2143. 2024-11-14T02:46:28,207 DEBUG [Thread-2143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:28,207 DEBUG [Thread-2143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:28,207 DEBUG [Thread-2143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:28,207 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:28,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T02:46:28,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:28,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:28,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:28,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T02:46:28,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:28,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:28,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:28,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,405 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/1e7528b3b3e84eb4b3a56a4c1d8ac841 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/1e7528b3b3e84eb4b3a56a4c1d8ac841 2024-11-14T02:46:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T02:46:28,409 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 672d111a762fbc594a3a447bdac7fae8/B of 672d111a762fbc594a3a447bdac7fae8 into 1e7528b3b3e84eb4b3a56a4c1d8ac841(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T02:46:28,409 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:28,409 INFO [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., storeName=672d111a762fbc594a3a447bdac7fae8/B, priority=13, startTime=1731552387958; duration=0sec 2024-11-14T02:46:28,409 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T02:46:28,409 DEBUG [RS:0;e5dc8b5cf5e9:46387-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 672d111a762fbc594a3a447bdac7fae8:B 2024-11-14T02:46:28,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552448415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:28,494 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/7d6dc6c7d77d4aba9a5e3c2327a68963 2024-11-14T02:46:28,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/135d39580ef94806bcfc82a453980d2e is 50, key is test_row_0/B:col10/1731552388047/Put/seqid=0 2024-11-14T02:46:28,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742405_1581 (size=12301) 2024-11-14T02:46:28,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T02:46:28,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:28,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:28,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:28,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T02:46:28,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:28,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:28,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:28,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552448725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:28,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T02:46:28,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:28,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:28,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:28,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T02:46:28,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/135d39580ef94806bcfc82a453980d2e 2024-11-14T02:46:28,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/320555814920449dbd30e013edb2bddb is 50, key is test_row_0/C:col10/1731552388047/Put/seqid=0 2024-11-14T02:46:28,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742406_1582 (size=12301) 2024-11-14T02:46:29,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T02:46:29,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:29,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:29,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:29,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:29,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:29,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T02:46:29,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:29,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:29,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:29,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:29,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:29,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T02:46:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57006 deadline: 1731552449235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 , the old value is region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T02:46:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., hostname=e5dc8b5cf5e9,46387,1731552206828, seqNum=5 because the exception is null or not the one we care about 2024-11-14T02:46:29,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T02:46:29,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:29,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:29,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:29,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:29,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:29,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/320555814920449dbd30e013edb2bddb 2024-11-14T02:46:29,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/7d6dc6c7d77d4aba9a5e3c2327a68963 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/7d6dc6c7d77d4aba9a5e3c2327a68963 2024-11-14T02:46:29,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/7d6dc6c7d77d4aba9a5e3c2327a68963, entries=200, sequenceid=399, filesize=39.0 K 2024-11-14T02:46:29,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/135d39580ef94806bcfc82a453980d2e as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/135d39580ef94806bcfc82a453980d2e 2024-11-14T02:46:29,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/135d39580ef94806bcfc82a453980d2e, entries=150, sequenceid=399, filesize=12.0 K 2024-11-14T02:46:29,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/320555814920449dbd30e013edb2bddb as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/320555814920449dbd30e013edb2bddb 2024-11-14T02:46:29,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/320555814920449dbd30e013edb2bddb, entries=150, sequenceid=399, filesize=12.0 K 2024-11-14T02:46:29,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 672d111a762fbc594a3a447bdac7fae8 in 1308ms, sequenceid=399, compaction requested=false 2024-11-14T02:46:29,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:29,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46387 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T02:46:29,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:29,461 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T02:46:29,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:29,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:29,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:29,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:29,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:29,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:29,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114dd462293171b45cd9f37f005bc24e5c4_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552388083/Put/seqid=0 2024-11-14T02:46:29,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742407_1583 (size=12454) 2024-11-14T02:46:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46387 {}] regionserver.HRegion(8855): Flush requested on 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:29,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. as already flushing 2024-11-14T02:46:29,597 INFO [Thread-2132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2132. 2024-11-14T02:46:29,597 DEBUG [Thread-2132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:29,597 DEBUG [Thread-2132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:29,597 INFO [Thread-2138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2138. 2024-11-14T02:46:29,597 DEBUG [Thread-2132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:29,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:29,597 DEBUG [Thread-2138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:29,597 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:29,597 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:29,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:29,607 INFO [Thread-2140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2140. 2024-11-14T02:46:29,607 DEBUG [Thread-2140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:29,607 DEBUG [Thread-2140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:29,607 DEBUG [Thread-2140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:29,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:29,626 INFO [Thread-2136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2136. 2024-11-14T02:46:29,626 DEBUG [Thread-2136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:29,627 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:29,627 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:29,627 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:29,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:29,873 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114dd462293171b45cd9f37f005bc24e5c4_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dd462293171b45cd9f37f005bc24e5c4_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:29,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/9115f634f4854099ad19efee273019da, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:29,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/9115f634f4854099ad19efee273019da is 175, key is test_row_0/A:col10/1731552388083/Put/seqid=0 2024-11-14T02:46:29,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742408_1584 (size=31255) 2024-11-14T02:46:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T02:46:30,246 INFO [Thread-2134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2134. 2024-11-14T02:46:30,246 DEBUG [Thread-2134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T02:46:30,246 DEBUG [Thread-2134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:30,246 DEBUG [Thread-2134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:30,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:30,278 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/9115f634f4854099ad19efee273019da 2024-11-14T02:46:30,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/26d0e0fa448a444f906aa62a281a1150 is 50, key is test_row_0/B:col10/1731552388083/Put/seqid=0 2024-11-14T02:46:30,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742409_1585 (size=12301) 2024-11-14T02:46:30,690 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/26d0e0fa448a444f906aa62a281a1150 2024-11-14T02:46:30,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/e6435247156c4c3582116bb4869b35ef is 50, key is test_row_0/C:col10/1731552388083/Put/seqid=0 2024-11-14T02:46:30,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742410_1586 (size=12301) 2024-11-14T02:46:31,099 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/e6435247156c4c3582116bb4869b35ef 2024-11-14T02:46:31,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/9115f634f4854099ad19efee273019da as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/9115f634f4854099ad19efee273019da 2024-11-14T02:46:31,106 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/9115f634f4854099ad19efee273019da, entries=150, sequenceid=413, filesize=30.5 K 2024-11-14T02:46:31,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/26d0e0fa448a444f906aa62a281a1150 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/26d0e0fa448a444f906aa62a281a1150 2024-11-14T02:46:31,110 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/26d0e0fa448a444f906aa62a281a1150, entries=150, sequenceid=413, filesize=12.0 K 2024-11-14T02:46:31,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/e6435247156c4c3582116bb4869b35ef as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/e6435247156c4c3582116bb4869b35ef 2024-11-14T02:46:31,114 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/e6435247156c4c3582116bb4869b35ef, entries=150, sequenceid=413, filesize=12.0 K 2024-11-14T02:46:31,115 INFO [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=33.54 KB/34350 for 672d111a762fbc594a3a447bdac7fae8 in 1654ms, sequenceid=413, compaction requested=true 2024-11-14T02:46:31,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 672d111a762fbc594a3a447bdac7fae8: 2024-11-14T02:46:31,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:31,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-14T02:46:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-14T02:46:31,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-14T02:46:31,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3360 sec 2024-11-14T02:46:31,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 3.3420 sec 2024-11-14T02:46:31,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T02:46:31,946 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:31,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T02:46:31,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-14T02:46:31,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-14T02:46:31,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-14T02:46:31,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-14T02:46:31,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-14T02:46:31,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T02:46:31,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2857 2024-11-14T02:46:31,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2769 2024-11-14T02:46:31,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2868 2024-11-14T02:46:31,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2854 2024-11-14T02:46:31,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2768 2024-11-14T02:46:31,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T02:46:31,946 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:46:31,947 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:31,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:31,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:31,947 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:31,947 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T02:46:31,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T02:46:31,950 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552391950"}]},"ts":"1731552391950"} 2024-11-14T02:46:31,952 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T02:46:31,952 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T02:46:31,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T02:46:31,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=672d111a762fbc594a3a447bdac7fae8, UNASSIGN}] 2024-11-14T02:46:31,955 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=672d111a762fbc594a3a447bdac7fae8, UNASSIGN 2024-11-14T02:46:31,956 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=672d111a762fbc594a3a447bdac7fae8, regionState=CLOSING, regionLocation=e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:31,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=672d111a762fbc594a3a447bdac7fae8, UNASSIGN because future has completed 2024-11-14T02:46:31,958 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T02:46:31,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828}] 2024-11-14T02:46:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T02:46:32,110 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(122): Close 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:32,110 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T02:46:32,111 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1722): Closing 672d111a762fbc594a3a447bdac7fae8, disabling compactions & flushes 2024-11-14T02:46:32,111 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:32,111 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:32,111 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. after waiting 0 ms 2024-11-14T02:46:32,111 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:32,111 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(2902): Flushing 672d111a762fbc594a3a447bdac7fae8 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T02:46:32,111 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=A 2024-11-14T02:46:32,111 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:32,111 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=B 2024-11-14T02:46:32,111 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:32,111 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 672d111a762fbc594a3a447bdac7fae8, store=C 2024-11-14T02:46:32,111 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T02:46:32,116 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b3df0d14922f482c920aab2632163e48_672d111a762fbc594a3a447bdac7fae8 is 50, key is test_row_0/A:col10/1731552389625/Put/seqid=0 2024-11-14T02:46:32,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742411_1587 (size=12454) 2024-11-14T02:46:32,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T02:46:32,522 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:32,530 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b3df0d14922f482c920aab2632163e48_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b3df0d14922f482c920aab2632163e48_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:32,531 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/890a1718c2894f848fb32d36a2fab726, store: [table=TestAcidGuarantees family=A region=672d111a762fbc594a3a447bdac7fae8] 2024-11-14T02:46:32,532 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/890a1718c2894f848fb32d36a2fab726 is 175, key is test_row_0/A:col10/1731552389625/Put/seqid=0 2024-11-14T02:46:32,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742412_1588 (size=31255) 2024-11-14T02:46:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T02:46:32,937 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/890a1718c2894f848fb32d36a2fab726 2024-11-14T02:46:32,943 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/bd6136292ee744738ff1efaef1a77ab5 is 50, key is test_row_0/B:col10/1731552389625/Put/seqid=0 2024-11-14T02:46:32,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742413_1589 (size=12301) 2024-11-14T02:46:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T02:46:33,347 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/bd6136292ee744738ff1efaef1a77ab5 2024-11-14T02:46:33,356 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/a12775b3cc494e33a664fec7756909e2 is 50, key is test_row_0/C:col10/1731552389625/Put/seqid=0 2024-11-14T02:46:33,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742414_1590 (size=12301) 2024-11-14T02:46:33,760 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/a12775b3cc494e33a664fec7756909e2 2024-11-14T02:46:33,763 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/A/890a1718c2894f848fb32d36a2fab726 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/890a1718c2894f848fb32d36a2fab726 2024-11-14T02:46:33,765 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/890a1718c2894f848fb32d36a2fab726, entries=150, sequenceid=421, filesize=30.5 K 2024-11-14T02:46:33,766 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/B/bd6136292ee744738ff1efaef1a77ab5 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/bd6136292ee744738ff1efaef1a77ab5 2024-11-14T02:46:33,768 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/bd6136292ee744738ff1efaef1a77ab5, entries=150, sequenceid=421, filesize=12.0 K 2024-11-14T02:46:33,769 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/.tmp/C/a12775b3cc494e33a664fec7756909e2 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/a12775b3cc494e33a664fec7756909e2 2024-11-14T02:46:33,771 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/a12775b3cc494e33a664fec7756909e2, entries=150, sequenceid=421, filesize=12.0 K 2024-11-14T02:46:33,772 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 672d111a762fbc594a3a447bdac7fae8 in 1661ms, sequenceid=421, compaction requested=true 2024-11-14T02:46:33,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/df554a3240454e0593af964f47297fe3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/ff0c6a726dd44b148d25ccf8a767b6c9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/0c74967c7e394b5389bec3c5598ab920, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/213513e88d1542c29a636dca01794148, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/b7a3ee6426994e65a2b595ac16be9d5c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/0931933e9fbc4e6c8217333c4d9e9628, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/e38fec9b634d4cfb92e1a86e2166509e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/d9cc1447410c44afacc694b0616e5a5f, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/ec69640f4c0940c5ab17c66133c41287, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/6321bdfd2574436aba08f2386a55b8ee, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/1e8100c2054c4ef7a6430d89ef82e84a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/cc094a2f5717472a983572a263d64f38, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/99c4031b3a73483d905a75efa4ae0a28, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/4ea2e70138a8472db755ad99c58c4d14, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/f856eaa9ad494d79b0e3477eaa227576, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/8024cb27701b4c4dbdb9ace055751eda, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/55deae69957549b9b6f9e9674c3f30d5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/774f4657bed04305bfdae7c91d63169b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/c86bcab4d8794d48abbe5149ba9d5d6d, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/64becffcfb4e4de3b5bf91c1ca16dde0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/edf75cb2a5fb4eafaf1c4f6d4867f9a8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/c676b08974084483a426945ee4dfcb95, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/b78a5325b88248678989471cff5250b1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/7c31807ed1b44766b58b5760d4207b3e, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/05d9c5bcda1c4cb089b28d631189c78a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/26ba2827fd244af59548f5dc86606065] to archive 2024-11-14T02:46:33,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:46:33,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/df554a3240454e0593af964f47297fe3 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/df554a3240454e0593af964f47297fe3 2024-11-14T02:46:33,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/ff0c6a726dd44b148d25ccf8a767b6c9 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/ff0c6a726dd44b148d25ccf8a767b6c9 2024-11-14T02:46:33,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/0c74967c7e394b5389bec3c5598ab920 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/0c74967c7e394b5389bec3c5598ab920 2024-11-14T02:46:33,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/213513e88d1542c29a636dca01794148 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/213513e88d1542c29a636dca01794148 2024-11-14T02:46:33,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/b7a3ee6426994e65a2b595ac16be9d5c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/b7a3ee6426994e65a2b595ac16be9d5c 2024-11-14T02:46:33,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/0931933e9fbc4e6c8217333c4d9e9628 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/0931933e9fbc4e6c8217333c4d9e9628 2024-11-14T02:46:33,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/e38fec9b634d4cfb92e1a86e2166509e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/e38fec9b634d4cfb92e1a86e2166509e 2024-11-14T02:46:33,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/d9cc1447410c44afacc694b0616e5a5f to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/d9cc1447410c44afacc694b0616e5a5f 2024-11-14T02:46:33,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/ec69640f4c0940c5ab17c66133c41287 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/ec69640f4c0940c5ab17c66133c41287 2024-11-14T02:46:33,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/6321bdfd2574436aba08f2386a55b8ee to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/6321bdfd2574436aba08f2386a55b8ee 2024-11-14T02:46:33,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/1e8100c2054c4ef7a6430d89ef82e84a to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/1e8100c2054c4ef7a6430d89ef82e84a 2024-11-14T02:46:33,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/cc094a2f5717472a983572a263d64f38 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/cc094a2f5717472a983572a263d64f38 2024-11-14T02:46:33,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/99c4031b3a73483d905a75efa4ae0a28 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/99c4031b3a73483d905a75efa4ae0a28 2024-11-14T02:46:33,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/4ea2e70138a8472db755ad99c58c4d14 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/4ea2e70138a8472db755ad99c58c4d14 2024-11-14T02:46:33,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/f856eaa9ad494d79b0e3477eaa227576 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/f856eaa9ad494d79b0e3477eaa227576 2024-11-14T02:46:33,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/8024cb27701b4c4dbdb9ace055751eda to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/8024cb27701b4c4dbdb9ace055751eda 2024-11-14T02:46:33,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/55deae69957549b9b6f9e9674c3f30d5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/55deae69957549b9b6f9e9674c3f30d5 2024-11-14T02:46:33,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/774f4657bed04305bfdae7c91d63169b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/774f4657bed04305bfdae7c91d63169b 2024-11-14T02:46:33,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/c86bcab4d8794d48abbe5149ba9d5d6d to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/c86bcab4d8794d48abbe5149ba9d5d6d 2024-11-14T02:46:33,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/64becffcfb4e4de3b5bf91c1ca16dde0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/64becffcfb4e4de3b5bf91c1ca16dde0 2024-11-14T02:46:33,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/edf75cb2a5fb4eafaf1c4f6d4867f9a8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/edf75cb2a5fb4eafaf1c4f6d4867f9a8 2024-11-14T02:46:33,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/c676b08974084483a426945ee4dfcb95 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/c676b08974084483a426945ee4dfcb95 2024-11-14T02:46:33,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/b78a5325b88248678989471cff5250b1 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/b78a5325b88248678989471cff5250b1 2024-11-14T02:46:33,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/7c31807ed1b44766b58b5760d4207b3e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/7c31807ed1b44766b58b5760d4207b3e 2024-11-14T02:46:33,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/05d9c5bcda1c4cb089b28d631189c78a to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/05d9c5bcda1c4cb089b28d631189c78a 2024-11-14T02:46:33,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/26ba2827fd244af59548f5dc86606065 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/26ba2827fd244af59548f5dc86606065 2024-11-14T02:46:33,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/c24927d1409a40748bf92bfe82685c97, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/edc595fb12104310a092f78282b94d6b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/d7a71e081c9b425baae4cdd04c2c3cba, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/15ef18ce5e3b46ef9a6c0d7480657bc3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/db08164ea5164d6fbe7c18fa146469f8, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/5494be801b6a4aa790c29035dc41621a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/87a320849bd3436c89fe6e7bf22ecaf5, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/95d287bd82f049f49c02d4ba9e118fa2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/771d400fd27d4480881872850b2a7f70, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a1752ecb0bc9478f8daa92054ec83e9b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/cd6b56e37ebb47d8aa278698e0900b49, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/f42c02b6276442209ec25053993db406, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/e682a9fef073406bb91e99a525895359, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/9be3e2576ac24e01830a5fe26e42d34c, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a15e889e8039463cae2f6cf2adb836b2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/31f6daab974d48b1b74ad20cf840e0d9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/501f873500c544089bacc3491ec42936, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b2402c4aff38443e95783b13dce079ea, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/0d125ea5203b4b258c274362e01671d6, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b182cfc037ee4de7af1a21c85152f4e7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/552e599e78f7442bb8ad6d958f07aaa3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a57b3c927c6c4e63bc4f4c0aacb231b9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/95370410e55a4460960e968ad19c4e7b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/25e8f14b2fef49acbc7b04358337c810, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/ad29154bf94d4b6497f0199c2027e48d, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b6f84a6fd0e54bc3a2e987f4b48cfd9e] to archive 2024-11-14T02:46:33,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:46:33,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/c24927d1409a40748bf92bfe82685c97 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/c24927d1409a40748bf92bfe82685c97 2024-11-14T02:46:33,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/edc595fb12104310a092f78282b94d6b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/edc595fb12104310a092f78282b94d6b 2024-11-14T02:46:33,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/d7a71e081c9b425baae4cdd04c2c3cba to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/d7a71e081c9b425baae4cdd04c2c3cba 2024-11-14T02:46:33,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/15ef18ce5e3b46ef9a6c0d7480657bc3 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/15ef18ce5e3b46ef9a6c0d7480657bc3 2024-11-14T02:46:33,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/db08164ea5164d6fbe7c18fa146469f8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/db08164ea5164d6fbe7c18fa146469f8 2024-11-14T02:46:33,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/5494be801b6a4aa790c29035dc41621a to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/5494be801b6a4aa790c29035dc41621a 2024-11-14T02:46:33,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/87a320849bd3436c89fe6e7bf22ecaf5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/87a320849bd3436c89fe6e7bf22ecaf5 2024-11-14T02:46:33,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/95d287bd82f049f49c02d4ba9e118fa2 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/95d287bd82f049f49c02d4ba9e118fa2 2024-11-14T02:46:33,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/771d400fd27d4480881872850b2a7f70 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/771d400fd27d4480881872850b2a7f70 2024-11-14T02:46:33,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a1752ecb0bc9478f8daa92054ec83e9b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a1752ecb0bc9478f8daa92054ec83e9b 2024-11-14T02:46:33,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/cd6b56e37ebb47d8aa278698e0900b49 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/cd6b56e37ebb47d8aa278698e0900b49 2024-11-14T02:46:33,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/f42c02b6276442209ec25053993db406 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/f42c02b6276442209ec25053993db406 2024-11-14T02:46:33,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/e682a9fef073406bb91e99a525895359 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/e682a9fef073406bb91e99a525895359 2024-11-14T02:46:33,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/9be3e2576ac24e01830a5fe26e42d34c to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/9be3e2576ac24e01830a5fe26e42d34c 2024-11-14T02:46:33,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a15e889e8039463cae2f6cf2adb836b2 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a15e889e8039463cae2f6cf2adb836b2 2024-11-14T02:46:33,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/31f6daab974d48b1b74ad20cf840e0d9 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/31f6daab974d48b1b74ad20cf840e0d9 2024-11-14T02:46:33,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/501f873500c544089bacc3491ec42936 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/501f873500c544089bacc3491ec42936 2024-11-14T02:46:33,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b2402c4aff38443e95783b13dce079ea to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b2402c4aff38443e95783b13dce079ea 2024-11-14T02:46:33,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/0d125ea5203b4b258c274362e01671d6 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/0d125ea5203b4b258c274362e01671d6 2024-11-14T02:46:33,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b182cfc037ee4de7af1a21c85152f4e7 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b182cfc037ee4de7af1a21c85152f4e7 2024-11-14T02:46:33,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/552e599e78f7442bb8ad6d958f07aaa3 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/552e599e78f7442bb8ad6d958f07aaa3 2024-11-14T02:46:33,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a57b3c927c6c4e63bc4f4c0aacb231b9 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/a57b3c927c6c4e63bc4f4c0aacb231b9 2024-11-14T02:46:33,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/95370410e55a4460960e968ad19c4e7b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/95370410e55a4460960e968ad19c4e7b 2024-11-14T02:46:33,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/25e8f14b2fef49acbc7b04358337c810 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/25e8f14b2fef49acbc7b04358337c810 2024-11-14T02:46:33,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/ad29154bf94d4b6497f0199c2027e48d to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/ad29154bf94d4b6497f0199c2027e48d 2024-11-14T02:46:33,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b6f84a6fd0e54bc3a2e987f4b48cfd9e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/b6f84a6fd0e54bc3a2e987f4b48cfd9e 2024-11-14T02:46:33,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/fb4039ebdc87407b951a8daf2ced6df1, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/5c014bf75f1241db9668ed345ff6b46b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/1a2115044fa7440ba687dfa52487ff83, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/97553bebb5ac4900b09e354e94d55c3b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/41fa531b3fd64e13bb62f8da67c7d5b0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/a304f04d82374f4c816d9cd00507e8e3, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/680eb7c574c4456d8b6cfabc82085705, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/63c80f872a1f40a7a827958b7c7013f0, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/3e42334a8d944d9cb37a63ef67607e80, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/884e80c65c3543c094125741f814bb31, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/66bef4ec5f284b22abea2cd033413525, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/22a01da3a4824ceb84703ccdf3c0653b, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/fe19e10b40cc4c928e78b28597dafbbb, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/77b511beecb444289a18e2f1c3981290, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/25fe503ef08f4e9087a33d4face8e8b7, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/eaf1ceebb79f49fd839be923b25ad790, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/aa7363128ac5497ba92032ffc268e067, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/77adc067e11545c3ad4bddf1df6d462a, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/8691d59a544b44fabc27bd1f15d97fc2, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/4c561d4b3c184c3a901125074bdb5858, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/ab3aeac06b6d4366bab9792d991acbb6, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/645a74eece014841a907ed1ba657cf08, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/07a532a4bb784bffb7c5cf4a396fc761, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/4910fd005b834f209c74c656e13a8454, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/0ff53a5d02314365bc9cea7b3d7746a9, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/2859441cd2c743a1b23ccceedec17b11] to archive 2024-11-14T02:46:33,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T02:46:33,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/fb4039ebdc87407b951a8daf2ced6df1 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/fb4039ebdc87407b951a8daf2ced6df1 2024-11-14T02:46:33,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/5c014bf75f1241db9668ed345ff6b46b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/5c014bf75f1241db9668ed345ff6b46b 2024-11-14T02:46:33,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/1a2115044fa7440ba687dfa52487ff83 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/1a2115044fa7440ba687dfa52487ff83 2024-11-14T02:46:33,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/97553bebb5ac4900b09e354e94d55c3b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/97553bebb5ac4900b09e354e94d55c3b 2024-11-14T02:46:33,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/41fa531b3fd64e13bb62f8da67c7d5b0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/41fa531b3fd64e13bb62f8da67c7d5b0 2024-11-14T02:46:33,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/a304f04d82374f4c816d9cd00507e8e3 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/a304f04d82374f4c816d9cd00507e8e3 2024-11-14T02:46:33,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/680eb7c574c4456d8b6cfabc82085705 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/680eb7c574c4456d8b6cfabc82085705 2024-11-14T02:46:33,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/63c80f872a1f40a7a827958b7c7013f0 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/63c80f872a1f40a7a827958b7c7013f0 2024-11-14T02:46:33,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/3e42334a8d944d9cb37a63ef67607e80 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/3e42334a8d944d9cb37a63ef67607e80 2024-11-14T02:46:33,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/884e80c65c3543c094125741f814bb31 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/884e80c65c3543c094125741f814bb31 2024-11-14T02:46:33,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/66bef4ec5f284b22abea2cd033413525 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/66bef4ec5f284b22abea2cd033413525 2024-11-14T02:46:33,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/22a01da3a4824ceb84703ccdf3c0653b to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/22a01da3a4824ceb84703ccdf3c0653b 2024-11-14T02:46:33,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/fe19e10b40cc4c928e78b28597dafbbb to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/fe19e10b40cc4c928e78b28597dafbbb 2024-11-14T02:46:33,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/77b511beecb444289a18e2f1c3981290 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/77b511beecb444289a18e2f1c3981290 2024-11-14T02:46:33,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/25fe503ef08f4e9087a33d4face8e8b7 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/25fe503ef08f4e9087a33d4face8e8b7 2024-11-14T02:46:33,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/eaf1ceebb79f49fd839be923b25ad790 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/eaf1ceebb79f49fd839be923b25ad790 2024-11-14T02:46:33,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/aa7363128ac5497ba92032ffc268e067 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/aa7363128ac5497ba92032ffc268e067 2024-11-14T02:46:33,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/77adc067e11545c3ad4bddf1df6d462a to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/77adc067e11545c3ad4bddf1df6d462a 2024-11-14T02:46:33,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/8691d59a544b44fabc27bd1f15d97fc2 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/8691d59a544b44fabc27bd1f15d97fc2 2024-11-14T02:46:33,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/4c561d4b3c184c3a901125074bdb5858 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/4c561d4b3c184c3a901125074bdb5858 2024-11-14T02:46:33,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/ab3aeac06b6d4366bab9792d991acbb6 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/ab3aeac06b6d4366bab9792d991acbb6 2024-11-14T02:46:33,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/645a74eece014841a907ed1ba657cf08 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/645a74eece014841a907ed1ba657cf08 2024-11-14T02:46:33,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/07a532a4bb784bffb7c5cf4a396fc761 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/07a532a4bb784bffb7c5cf4a396fc761 2024-11-14T02:46:33,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/4910fd005b834f209c74c656e13a8454 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/4910fd005b834f209c74c656e13a8454 2024-11-14T02:46:33,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/0ff53a5d02314365bc9cea7b3d7746a9 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/0ff53a5d02314365bc9cea7b3d7746a9 2024-11-14T02:46:33,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/2859441cd2c743a1b23ccceedec17b11 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/2859441cd2c743a1b23ccceedec17b11 2024-11-14T02:46:33,843 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/recovered.edits/424.seqid, newMaxSeqId=424, maxSeqId=4 2024-11-14T02:46:33,844 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. 2024-11-14T02:46:33,844 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1676): Region close journal for 672d111a762fbc594a3a447bdac7fae8: Waiting for close lock at 1731552392110Running coprocessor pre-close hooks at 1731552392110Disabling compacts and flushes for region at 1731552392110Disabling writes for close at 1731552392111 (+1 ms)Obtaining lock to block concurrent updates at 1731552392111Preparing flush snapshotting stores in 672d111a762fbc594a3a447bdac7fae8 at 1731552392111Finished memstore snapshotting TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731552392111Flushing stores of TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8. at 1731552392111Flushing 672d111a762fbc594a3a447bdac7fae8/A: creating writer at 1731552392111Flushing mob file 672d111a762fbc594a3a447bdac7fae8/A: appending metadata at 1731552392116 (+5 ms)Flushing mob file 672d111a762fbc594a3a447bdac7fae8/A: closing flushed file at 1731552392116Flushing 672d111a762fbc594a3a447bdac7fae8/A: appending metadata at 1731552392531 (+415 ms)Flushing 672d111a762fbc594a3a447bdac7fae8/A: closing flushed file at 1731552392531Flushing 672d111a762fbc594a3a447bdac7fae8/B: creating writer at 1731552392940 (+409 ms)Flushing 672d111a762fbc594a3a447bdac7fae8/B: appending metadata at 1731552392943 (+3 ms)Flushing 672d111a762fbc594a3a447bdac7fae8/B: closing flushed file at 1731552392943Flushing 672d111a762fbc594a3a447bdac7fae8/C: creating writer at 1731552393352 (+409 ms)Flushing 672d111a762fbc594a3a447bdac7fae8/C: appending metadata at 1731552393355 (+3 ms)Flushing 672d111a762fbc594a3a447bdac7fae8/C: closing flushed file at 1731552393355Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fbddf18: reopening flushed file at 1731552393763 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a8609f6: reopening flushed file at 1731552393765 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29b9aaa: reopening flushed file at 1731552393768 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 672d111a762fbc594a3a447bdac7fae8 in 1661ms, sequenceid=421, compaction requested=true at 1731552393772 (+4 ms)Writing region close event to WAL at 1731552393841 (+69 ms)Running coprocessor post-close hooks at 1731552393844 (+3 ms)Closed at 1731552393844 2024-11-14T02:46:33,845 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(157): Closed 672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:33,846 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=672d111a762fbc594a3a447bdac7fae8, regionState=CLOSED 2024-11-14T02:46:33,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 because future has completed 2024-11-14T02:46:33,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-14T02:46:33,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseRegionProcedure 672d111a762fbc594a3a447bdac7fae8, server=e5dc8b5cf5e9,46387,1731552206828 in 1.8890 sec 2024-11-14T02:46:33,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-14T02:46:33,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=672d111a762fbc594a3a447bdac7fae8, UNASSIGN in 1.8940 sec 2024-11-14T02:46:33,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-14T02:46:33,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8970 sec 2024-11-14T02:46:33,852 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552393852"}]},"ts":"1731552393852"} 2024-11-14T02:46:33,853 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T02:46:33,853 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T02:46:33,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9060 sec 2024-11-14T02:46:34,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T02:46:34,096 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:34,097 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T02:46:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:34,100 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:34,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T02:46:34,102 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=162, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:34,105 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,106 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C, FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/recovered.edits] 2024-11-14T02:46:34,108 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/161706cac46142c4ac74b4e77d8ed074 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/161706cac46142c4ac74b4e77d8ed074 2024-11-14T02:46:34,109 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/7d6dc6c7d77d4aba9a5e3c2327a68963 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/7d6dc6c7d77d4aba9a5e3c2327a68963 2024-11-14T02:46:34,110 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/890a1718c2894f848fb32d36a2fab726 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/890a1718c2894f848fb32d36a2fab726 2024-11-14T02:46:34,111 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/9115f634f4854099ad19efee273019da to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/A/9115f634f4854099ad19efee273019da 2024-11-14T02:46:34,112 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/135d39580ef94806bcfc82a453980d2e to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/135d39580ef94806bcfc82a453980d2e 2024-11-14T02:46:34,113 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/1e7528b3b3e84eb4b3a56a4c1d8ac841 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/1e7528b3b3e84eb4b3a56a4c1d8ac841 2024-11-14T02:46:34,113 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/26d0e0fa448a444f906aa62a281a1150 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/26d0e0fa448a444f906aa62a281a1150 2024-11-14T02:46:34,114 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/bd6136292ee744738ff1efaef1a77ab5 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/B/bd6136292ee744738ff1efaef1a77ab5 2024-11-14T02:46:34,116 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/320555814920449dbd30e013edb2bddb to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/320555814920449dbd30e013edb2bddb 2024-11-14T02:46:34,117 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/5bf69f3d90324e95ae87a7a4e074fa92 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/5bf69f3d90324e95ae87a7a4e074fa92 2024-11-14T02:46:34,118 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/a12775b3cc494e33a664fec7756909e2 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/a12775b3cc494e33a664fec7756909e2 2024-11-14T02:46:34,119 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/e6435247156c4c3582116bb4869b35ef to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/C/e6435247156c4c3582116bb4869b35ef 2024-11-14T02:46:34,120 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/recovered.edits/424.seqid to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8/recovered.edits/424.seqid 2024-11-14T02:46:34,121 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/default/TestAcidGuarantees/672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,121 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T02:46:34,121 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T02:46:34,122 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T02:46:34,124 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111402188353d7734e7cb6b3ac1020b45e32_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111402188353d7734e7cb6b3ac1020b45e32_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,125 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111402cae0d24c7541b1bd34dea8722752dc_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111402cae0d24c7541b1bd34dea8722752dc_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,126 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114170ae0d28746408e84adfb6592aa3a23_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114170ae0d28746408e84adfb6592aa3a23_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,126 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141a0699942d704ed8b16d11840e183367_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141a0699942d704ed8b16d11840e183367_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,127 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141f7299b58253492bb19117a1871b7d86_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141f7299b58253492bb19117a1871b7d86_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,128 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111443840b8e702f4d05a7e42f339dd7e071_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111443840b8e702f4d05a7e42f339dd7e071_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,129 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144df6e08b582c4b22b39e84ecc201f025_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144df6e08b582c4b22b39e84ecc201f025_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,129 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145282da8fe01f40f6a276b7734edc2ede_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145282da8fe01f40f6a276b7734edc2ede_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,130 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b10a6e325cfd44ab9df6f02dc6dfe659_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b10a6e325cfd44ab9df6f02dc6dfe659_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,131 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b1a63357e2414c619f012bb20e2b799e_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b1a63357e2414c619f012bb20e2b799e_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,131 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b3df0d14922f482c920aab2632163e48_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b3df0d14922f482c920aab2632163e48_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,132 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bbc00af0dc94405184253f8c9a3249be_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bbc00af0dc94405184253f8c9a3249be_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,133 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c19075ee44e74291862cc20eb9928eee_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c19075ee44e74291862cc20eb9928eee_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,134 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c3531bfbfd1d4cf6989d1b0adedb0cbe_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c3531bfbfd1d4cf6989d1b0adedb0cbe_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,134 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d8811f33b4ea4b8e82e9162476e58d46_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d8811f33b4ea4b8e82e9162476e58d46_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,135 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114da90bf6c3fa14218baf08e5ddf664445_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114da90bf6c3fa14218baf08e5ddf664445_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,135 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dd462293171b45cd9f37f005bc24e5c4_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dd462293171b45cd9f37f005bc24e5c4_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,136 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e4bc35c6854a4ce186e4f67c11be6496_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e4bc35c6854a4ce186e4f67c11be6496_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,137 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e51a60cc9a0948498bf7626b2bc9d8f5_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e51a60cc9a0948498bf7626b2bc9d8f5_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,137 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ead9d05da0ce474298d97b1767343523_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ead9d05da0ce474298d97b1767343523_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,138 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f5d9fa7141324c73bcb432c6cabee225_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f5d9fa7141324c73bcb432c6cabee225_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,139 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fb5291d9958248e2bf3b223a59ac5f01_672d111a762fbc594a3a447bdac7fae8 to hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fb5291d9958248e2bf3b223a59ac5f01_672d111a762fbc594a3a447bdac7fae8 2024-11-14T02:46:34,139 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T02:46:34,140 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=162, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:34,142 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T02:46:34,144 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T02:46:34,144 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=162, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:34,144 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T02:46:34,145 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731552394144"}]},"ts":"9223372036854775807"} 2024-11-14T02:46:34,146 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T02:46:34,146 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 672d111a762fbc594a3a447bdac7fae8, NAME => 'TestAcidGuarantees,,1731552365023.672d111a762fbc594a3a447bdac7fae8.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T02:46:34,146 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T02:46:34,146 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731552394146"}]},"ts":"9223372036854775807"} 2024-11-14T02:46:34,147 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T02:46:34,148 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=162, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T02:46:34,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-14T02:46:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38717 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T02:46:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T02:46:34,205 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T02:46:34,212 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250 (was 254), OpenFileDescriptor=451 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=437 (was 323) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9507 (was 9687) 2024-11-14T02:46:34,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T02:46:34,213 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:46:34,213 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:34,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:34,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:34,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:34,213 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T02:46:34,213 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=841754823, stopped=false 2024-11-14T02:46:34,213 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e5dc8b5cf5e9,38717,1731552206127 2024-11-14T02:46:34,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:34,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:34,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:34,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:34,232 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T02:46:34,232 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:46:34,232 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:34,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:34,233 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:34,233 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:34,233 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e5dc8b5cf5e9,46387,1731552206828' ***** 2024-11-14T02:46:34,233 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T02:46:34,233 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T02:46:34,234 INFO [RS:0;e5dc8b5cf5e9:46387 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T02:46:34,234 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T02:46:34,234 INFO [RS:0;e5dc8b5cf5e9:46387 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T02:46:34,234 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(959): stopping server e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:34,234 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T02:46:34,234 INFO [RS:0;e5dc8b5cf5e9:46387 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e5dc8b5cf5e9:46387. 2024-11-14T02:46:34,234 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:34,234 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:34,234 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T02:46:34,234 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T02:46:34,234 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T02:46:34,234 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T02:46:34,235 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T02:46:34,235 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T02:46:34,235 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:34,235 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T02:46:34,235 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T02:46:34,235 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T02:46:34,235 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T02:46:34,235 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T02:46:34,235 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-14T02:46:34,265 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/.tmp/info/12a5ae945f7b45ed97588fb9abab24d4 is 95, key is TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb./info:/1731552244673/DeleteFamily/seqid=0 2024-11-14T02:46:34,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742415_1591 (size=6514) 2024-11-14T02:46:34,311 INFO [regionserver/e5dc8b5cf5e9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T02:46:34,435 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:34,586 INFO [regionserver/e5dc8b5cf5e9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T02:46:34,586 INFO [regionserver/e5dc8b5cf5e9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T02:46:34,635 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:34,668 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/.tmp/info/12a5ae945f7b45ed97588fb9abab24d4 2024-11-14T02:46:34,685 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/.tmp/ns/ae06b09ef2914a9e9182a7ab42659175 is 93, key is TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb./ns:/1731552244632/DeleteFamily/seqid=0 2024-11-14T02:46:34,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742416_1592 (size=5906) 2024-11-14T02:46:34,836 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:35,036 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:35,089 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/.tmp/ns/ae06b09ef2914a9e9182a7ab42659175 2024-11-14T02:46:35,112 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/.tmp/rep_barrier/741d2c933a7b41bf95fc36eafa6fdcda is 102, key is TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb./rep_barrier:/1731552244632/DeleteFamily/seqid=0 2024-11-14T02:46:35,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742417_1593 (size=6025) 2024-11-14T02:46:35,236 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:35,437 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:35,517 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/.tmp/rep_barrier/741d2c933a7b41bf95fc36eafa6fdcda 2024-11-14T02:46:35,541 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/.tmp/table/fd62749186bd491cbc250f0ba8de015f is 96, key is TestAcidGuarantees,,1731552211260.b0f2fe4f29c62d92b3ee0b2d165c19cb./table:/1731552244632/DeleteFamily/seqid=0 2024-11-14T02:46:35,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742418_1594 (size=5936) 2024-11-14T02:46:35,637 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:35,837 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:35,945 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/.tmp/table/fd62749186bd491cbc250f0ba8de015f 2024-11-14T02:46:35,951 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/.tmp/info/12a5ae945f7b45ed97588fb9abab24d4 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/info/12a5ae945f7b45ed97588fb9abab24d4 2024-11-14T02:46:35,955 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/info/12a5ae945f7b45ed97588fb9abab24d4, entries=12, sequenceid=90, filesize=6.4 K 2024-11-14T02:46:35,956 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/.tmp/ns/ae06b09ef2914a9e9182a7ab42659175 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/ns/ae06b09ef2914a9e9182a7ab42659175 2024-11-14T02:46:35,960 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/ns/ae06b09ef2914a9e9182a7ab42659175, entries=8, sequenceid=90, filesize=5.8 K 2024-11-14T02:46:35,960 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/.tmp/rep_barrier/741d2c933a7b41bf95fc36eafa6fdcda as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/rep_barrier/741d2c933a7b41bf95fc36eafa6fdcda 2024-11-14T02:46:35,963 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/rep_barrier/741d2c933a7b41bf95fc36eafa6fdcda, entries=6, sequenceid=90, filesize=5.9 K 2024-11-14T02:46:35,964 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/.tmp/table/fd62749186bd491cbc250f0ba8de015f as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/table/fd62749186bd491cbc250f0ba8de015f 2024-11-14T02:46:35,967 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/table/fd62749186bd491cbc250f0ba8de015f, entries=7, sequenceid=90, filesize=5.8 K 2024-11-14T02:46:35,967 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1732ms, sequenceid=90, compaction requested=false 2024-11-14T02:46:35,973 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-14T02:46:35,974 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T02:46:35,974 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T02:46:35,974 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731552394235Running coprocessor pre-close hooks at 1731552394235Disabling compacts and flushes for region at 1731552394235Disabling writes for close at 1731552394235Obtaining lock to block concurrent updates at 1731552394235Preparing flush snapshotting stores in 1588230740 at 1731552394235Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731552394236 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731552394236Flushing 1588230740/info: creating writer at 1731552394236Flushing 1588230740/info: appending metadata at 1731552394264 (+28 ms)Flushing 1588230740/info: closing flushed file at 1731552394264Flushing 1588230740/ns: creating writer at 1731552394672 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731552394685 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731552394685Flushing 1588230740/rep_barrier: creating writer at 1731552395098 (+413 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731552395112 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731552395112Flushing 1588230740/table: creating writer at 1731552395523 (+411 ms)Flushing 1588230740/table: appending metadata at 1731552395541 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731552395541Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a44e31a: reopening flushed file at 1731552395950 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a409d90: reopening flushed file at 1731552395955 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46c94e8f: reopening flushed file at 1731552395960 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29ab039e: reopening flushed file at 1731552395964 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1732ms, sequenceid=90, compaction requested=false at 1731552395967 (+3 ms)Writing region close event to WAL at 1731552395970 (+3 ms)Running coprocessor post-close hooks at 1731552395974 (+4 ms)Closed at 1731552395974 2024-11-14T02:46:35,974 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T02:46:36,037 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(976): stopping server e5dc8b5cf5e9,46387,1731552206828; all regions closed. 2024-11-14T02:46:36,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741834_1010 (size=25344) 2024-11-14T02:46:36,044 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/oldWALs 2024-11-14T02:46:36,044 INFO [RS:0;e5dc8b5cf5e9:46387 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e5dc8b5cf5e9%2C46387%2C1731552206828.meta:.meta(num 1731552210693) 2024-11-14T02:46:36,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741832_1008 (size=18261409) 2024-11-14T02:46:36,047 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/oldWALs 2024-11-14T02:46:36,047 INFO [RS:0;e5dc8b5cf5e9:46387 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e5dc8b5cf5e9%2C46387%2C1731552206828:(num 1731552209679) 2024-11-14T02:46:36,047 DEBUG [RS:0;e5dc8b5cf5e9:46387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:36,047 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T02:46:36,048 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T02:46:36,048 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.ChoreService(370): Chore service for: regionserver/e5dc8b5cf5e9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T02:46:36,048 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T02:46:36,048 INFO [regionserver/e5dc8b5cf5e9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T02:46:36,049 INFO [RS:0;e5dc8b5cf5e9:46387 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46387 2024-11-14T02:46:36,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T02:46:36,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e5dc8b5cf5e9,46387,1731552206828 2024-11-14T02:46:36,106 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T02:46:36,169 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e5dc8b5cf5e9,46387,1731552206828] 2024-11-14T02:46:36,264 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e5dc8b5cf5e9,46387,1731552206828 already deleted, retry=false 2024-11-14T02:46:36,265 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e5dc8b5cf5e9,46387,1731552206828 expired; onlineServers=0 2024-11-14T02:46:36,266 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e5dc8b5cf5e9,38717,1731552206127' ***** 2024-11-14T02:46:36,266 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T02:46:36,266 INFO [M:0;e5dc8b5cf5e9:38717 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T02:46:36,267 INFO [M:0;e5dc8b5cf5e9:38717 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T02:46:36,267 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T02:46:36,267 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T02:46:36,267 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T02:46:36,267 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.large.0-1731552209327 {}] cleaner.HFileCleaner(306): Exit Thread[master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.large.0-1731552209327,5,FailOnTimeoutGroup] 2024-11-14T02:46:36,267 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.small.0-1731552209328 {}] cleaner.HFileCleaner(306): Exit Thread[master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.small.0-1731552209328,5,FailOnTimeoutGroup] 2024-11-14T02:46:36,268 INFO [M:0;e5dc8b5cf5e9:38717 {}] hbase.ChoreService(370): Chore service for: master/e5dc8b5cf5e9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T02:46:36,269 INFO [M:0;e5dc8b5cf5e9:38717 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T02:46:36,269 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] master.HMaster(1795): Stopping service threads 2024-11-14T02:46:36,269 INFO [M:0;e5dc8b5cf5e9:38717 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T02:46:36,269 INFO [M:0;e5dc8b5cf5e9:38717 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T02:46:36,270 ERROR [M:0;e5dc8b5cf5e9:38717 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-14T02:46:36,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:36,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46387-0x1013739d5650001, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:36,271 INFO [RS:0;e5dc8b5cf5e9:46387 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T02:46:36,271 INFO [M:0;e5dc8b5cf5e9:38717 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T02:46:36,271 INFO [RS:0;e5dc8b5cf5e9:46387 {}] regionserver.HRegionServer(1031): Exiting; stopping=e5dc8b5cf5e9,46387,1731552206828; zookeeper connection closed. 2024-11-14T02:46:36,271 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T02:46:36,271 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@70a67ef3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@70a67ef3 2024-11-14T02:46:36,272 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T02:46:36,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T02:46:36,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:36,317 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] zookeeper.ZKUtil(347): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T02:46:36,317 WARN [M:0;e5dc8b5cf5e9:38717 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T02:46:36,318 INFO [M:0;e5dc8b5cf5e9:38717 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/.lastflushedseqids 2024-11-14T02:46:36,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742419_1595 (size=72) 2024-11-14T02:46:36,489 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T02:46:36,490 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T02:46:36,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T02:46:36,732 INFO [M:0;e5dc8b5cf5e9:38717 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T02:46:36,733 INFO [M:0;e5dc8b5cf5e9:38717 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T02:46:36,733 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T02:46:36,733 INFO [M:0;e5dc8b5cf5e9:38717 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:36,733 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:36,733 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T02:46:36,733 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:36,733 INFO [M:0;e5dc8b5cf5e9:38717 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=763.15 KB heapSize=934.38 KB 2024-11-14T02:46:36,755 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/83f16dcfddcf4c3dbce4f875c920d5dc is 82, key is hbase:meta,,1/info:regioninfo/1731552210769/Put/seqid=0 2024-11-14T02:46:36,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742420_1596 (size=5672) 2024-11-14T02:46:37,158 INFO [M:0;e5dc8b5cf5e9:38717 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2117 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/83f16dcfddcf4c3dbce4f875c920d5dc 2024-11-14T02:46:37,187 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e5b85d3429e64b97855816f29aac76ce is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x89/proc:d/1731552368113/Put/seqid=0 2024-11-14T02:46:37,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742421_1597 (size=43962) 2024-11-14T02:46:37,594 INFO [M:0;e5dc8b5cf5e9:38717 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=762.60 KB at sequenceid=2117 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e5b85d3429e64b97855816f29aac76ce 2024-11-14T02:46:37,597 INFO [M:0;e5dc8b5cf5e9:38717 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e5b85d3429e64b97855816f29aac76ce 2024-11-14T02:46:37,615 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ff1a9575ec834df18f289f7b4f222092 is 69, key is e5dc8b5cf5e9,46387,1731552206828/rs:state/1731552209452/Put/seqid=0 2024-11-14T02:46:37,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073742422_1598 (size=5156) 2024-11-14T02:46:38,019 INFO [M:0;e5dc8b5cf5e9:38717 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2117 (bloomFilter=true), to=hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ff1a9575ec834df18f289f7b4f222092 2024-11-14T02:46:38,023 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/83f16dcfddcf4c3dbce4f875c920d5dc as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/83f16dcfddcf4c3dbce4f875c920d5dc 2024-11-14T02:46:38,028 INFO [M:0;e5dc8b5cf5e9:38717 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/83f16dcfddcf4c3dbce4f875c920d5dc, entries=8, sequenceid=2117, filesize=5.5 K 2024-11-14T02:46:38,029 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e5b85d3429e64b97855816f29aac76ce as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e5b85d3429e64b97855816f29aac76ce 2024-11-14T02:46:38,033 INFO [M:0;e5dc8b5cf5e9:38717 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e5b85d3429e64b97855816f29aac76ce 2024-11-14T02:46:38,033 INFO [M:0;e5dc8b5cf5e9:38717 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e5b85d3429e64b97855816f29aac76ce, entries=162, sequenceid=2117, filesize=42.9 K 2024-11-14T02:46:38,034 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ff1a9575ec834df18f289f7b4f222092 as hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ff1a9575ec834df18f289f7b4f222092 2024-11-14T02:46:38,037 INFO [M:0;e5dc8b5cf5e9:38717 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34671/user/jenkins/test-data/f1981a06-de9a-6554-7e37-d997b41384f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ff1a9575ec834df18f289f7b4f222092, entries=1, sequenceid=2117, filesize=5.0 K 2024-11-14T02:46:38,038 INFO [M:0;e5dc8b5cf5e9:38717 {}] regionserver.HRegion(3140): Finished flush of dataSize ~763.15 KB/781467, heapSize ~934.08 KB/956496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1305ms, sequenceid=2117, compaction requested=false 2024-11-14T02:46:38,039 INFO [M:0;e5dc8b5cf5e9:38717 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:38,039 DEBUG [M:0;e5dc8b5cf5e9:38717 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731552396733Disabling compacts and flushes for region at 1731552396733Disabling writes for close at 1731552396733Obtaining lock to block concurrent updates at 1731552396733Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731552396733Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=781467, getHeapSize=956736, getOffHeapSize=0, getCellsCount=2304 at 1731552396734 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731552396735 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731552396735Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731552396754 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731552396754Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731552397164 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731552397187 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731552397187Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731552397597 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731552397614 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731552397614Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75847c88: reopening flushed file at 1731552398023 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f9b3927: reopening flushed file at 1731552398028 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2faa181: reopening flushed file at 1731552398033 (+5 ms)Finished flush of dataSize ~763.15 KB/781467, heapSize ~934.08 KB/956496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1305ms, sequenceid=2117, compaction requested=false at 1731552398038 (+5 ms)Writing region close event to WAL at 1731552398039 (+1 ms)Closed at 1731552398039 2024-11-14T02:46:38,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41091 is added to blk_1073741830_1006 (size=920239) 2024-11-14T02:46:38,042 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T02:46:38,042 INFO [M:0;e5dc8b5cf5e9:38717 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T02:46:38,042 INFO [M:0;e5dc8b5cf5e9:38717 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38717 2024-11-14T02:46:38,043 INFO [M:0;e5dc8b5cf5e9:38717 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T02:46:38,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:38,153 INFO [M:0;e5dc8b5cf5e9:38717 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T02:46:38,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38717-0x1013739d5650000, quorum=127.0.0.1:62229, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:38,164 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T02:46:38,167 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T02:46:38,167 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T02:46:38,167 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T02:46:38,167 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/hadoop.log.dir/,STOPPED} 2024-11-14T02:46:38,169 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T02:46:38,170 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T02:46:38,170 WARN [BP-1718068783-172.17.0.2-1731552202398 heartbeating to localhost/127.0.0.1:34671 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T02:46:38,170 WARN [BP-1718068783-172.17.0.2-1731552202398 heartbeating to localhost/127.0.0.1:34671 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1718068783-172.17.0.2-1731552202398 (Datanode Uuid a1e01ff2-a378-43eb-b94e-1163f3e76400) service to localhost/127.0.0.1:34671 2024-11-14T02:46:38,172 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/cluster_d5add8ed-0ad9-3315-92f0-5e7efe2147ab/data/data1/current/BP-1718068783-172.17.0.2-1731552202398 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T02:46:38,172 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/cluster_d5add8ed-0ad9-3315-92f0-5e7efe2147ab/data/data2/current/BP-1718068783-172.17.0.2-1731552202398 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T02:46:38,173 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T02:46:38,183 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T02:46:38,184 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T02:46:38,184 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T02:46:38,184 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T02:46:38,184 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a187d6ab-f18b-72ac-3bae-b00a4ca09802/hadoop.log.dir/,STOPPED} 2024-11-14T02:46:38,198 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T02:46:38,323 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down